From 4a70a93db8a302755c4c61ea1aa8ba8a39866b21 Mon Sep 17 00:00:00 2001 From: Eric Gao Date: Wed, 2 Nov 2022 23:06:43 +0800 Subject: [PATCH] [fix][sql] Add unique key to process_definition_log avoid TooManyResultExpection (#12503) (#12670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add unique key to t_ds_process_definition_log Co-authored-by: 陈家名 <13774486042@163.com> --- .../resources/sql/dolphinscheduler_h2.sql | 3 +- .../resources/sql/dolphinscheduler_mysql.sql | 3 +- .../sql/dolphinscheduler_postgresql.sql | 2 + .../mysql/dolphinscheduler_ddl.sql | 38 ++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++ .../postgresql/dolphinscheduler_ddl.sql | 19 +++ .../postgresql/dolphinscheduler_dml.sql | 16 +++ .../mysql/dolphinscheduler_ddl.sql | 122 ++++++++++++++++++ .../postgresql/dolphinscheduler_ddl.sql | 121 +++++++++++++++++ 9 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 9b69fb1c75..91b7c14684 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -456,7 +456,8 @@ CREATE TABLE t_ds_process_definition_log operate_time datetime DEFAULT NULL, create_time datetime NOT NULL, update_time datetime DEFAULT NULL, - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY uniq_idx_code_version (code, version) USING BTREE ); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 943d129531..0e54b37f63 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -457,7 +457,8 @@ CREATE TABLE `t_ds_process_definition_log` ( `operate_time` datetime DEFAULT NULL COMMENT 'operate time', `create_time` datetime NOT NULL COMMENT 'create time', `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_idx_code_version` (`code`,`version`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 0389d88dca..e89841746b 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -376,6 +376,8 @@ CREATE TABLE t_ds_process_definition_log ( PRIMARY KEY (id) ) ; +create UNIQUE index uniq_idx_code_version on t_ds_process_definition_log (code,version); + -- -- Table structure for table t_ds_task_definition -- diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..13894fd3c2 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + + +-- add unique key to t_ds_process_definition_log +drop PROCEDURE if EXISTS add_t_ds_process_definition_log_uk_uniq_idx_code_version; +delimiter d// +CREATE PROCEDURE add_t_ds_process_definition_log_uk_uniq_idx_code_version() +BEGIN + IF NOT EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_NAME='t_ds_process_definition_log' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND INDEX_NAME='uniq_idx_code_version') + THEN +ALTER TABLE t_ds_process_definition_log ADD UNIQUE KEY uniq_idx_code_version(`code`,`version`); +END IF; +END; + +d// + +delimiter ; +CALL add_t_ds_process_definition_log_uk_uniq_idx_code_version; +DROP PROCEDURE add_t_ds_process_definition_log_uk_uniq_idx_code_version; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..a691a02a6c --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +-- add unique key to t_ds_process_definition_log +create UNIQUE index IF NOT EXISTS uniq_idx_code_version on t_ds_process_definition_log (code,version); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..d852683640 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + +-- uc_dolphin_T_t_ds_command_R_test_flag +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_command_R_test_flag; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_command_R_test_flag() +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_command ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 test run'; +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_command_R_test_flag; +DROP PROCEDURE uc_dolphin_T_t_ds_command_R_test_flag; + +-- uc_dolphin_T_t_ds_error_command_R_test_flag +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_error_command_R_test_flag; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_error_command_R_test_flag() +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_error_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_error_command ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 test run'; +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_error_command_R_test_flag; +DROP PROCEDURE uc_dolphin_T_t_ds_error_command_R_test_flag; + +-- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id() +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_datasource' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='test_flag') + and NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_datasource' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='bind_test_id') + THEN +ALTER TABLE t_ds_datasource ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 testDataSource'; +ALTER TABLE t_ds_datasource ADD `bind_test_id` int DEFAULT null COMMENT 'bind testDataSource id'; +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id; +DROP PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id; + +-- uc_dolphin_T_t_ds_process_instance_R_test_flag +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_process_instance_R_test_flag() +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_process_instance ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 test run'; +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_process_instance_R_test_flag; +DROP PROCEDURE uc_dolphin_T_t_ds_process_instance_R_test_flag; + +-- uc_dolphin_T_t_ds_task_instance_R_test_flag +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_R_test_flag; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_R_test_flag() +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_task_instance ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 test run'; +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_R_test_flag; +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_R_test_flag; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..8b6dbba36b --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +-- uc_dolphin_T_t_ds_command_R_test_flag +delimiter ; +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_command_R_test_flag(); +delimiter d// +CREATE FUNCTION uc_dolphin_T_t_ds_command_R_test_flag() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_command' + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_command alter column test_flag type int DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select uc_dolphin_T_t_ds_command_R_test_flag(); +DROP FUNCTION uc_dolphin_T_t_ds_command_R_test_flag(); + +-- uc_dolphin_T_t_ds_error_command_R_test_flag +delimiter ; +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_error_command_R_test_flag(); +delimiter d// +CREATE FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_error_command' + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_error_command alter column test_flag type int DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select uc_dolphin_T_t_ds_error_command_R_test_flag(); +DROP FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag(); + +-- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id +delimiter ; +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id(); +delimiter d// +CREATE FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_datasource' + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_datasource alter column test_flag type int DEFAULT NULL; +ALTER TABLE t_ds_datasource alter column bind_test_id type int DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id(); +DROP FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id(); + +-- uc_dolphin_T_t_ds_process_instance_R_test_flag +delimiter ; +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag(); +delimiter d// +CREATE FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_process_instance' + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_process_instance alter column test_flag type int DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select uc_dolphin_T_t_ds_process_instance_R_test_flag(); +DROP FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag(); + +-- uc_dolphin_T_t_ds_task_instance_R_test_flag +delimiter ; +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_R_test_flag(); +delimiter d// +CREATE FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='test_flag') + THEN +ALTER TABLE t_ds_task_instance alter column test_flag type int DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select uc_dolphin_T_t_ds_task_instance_R_test_flag(); +DROP FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag();