From 2b5f8433b732eee8c31a05f741636a007d5f2344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=A7=E5=9F=8E?= Date: Thu, 11 Apr 2024 14:13:53 +0800 Subject: [PATCH 01/41] Fix cannot construct instance of StreamingTaskTriggerResponse --- .../master/transportor/StreamingTaskTriggerResponse.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java index 0f9f265280..d25611aee0 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java @@ -19,9 +19,11 @@ package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class StreamingTaskTriggerResponse { private boolean success; From 883848f6ee00843d3c9e4668d607336d544dd110 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 11 Apr 2024 19:28:00 +0800 Subject: [PATCH 02/41] Remove unused caffeine cache (#15830) --- dolphinscheduler-master/pom.xml | 4 ---- .../src/main/resources/application.yaml | 11 ----------- .../src/test/resources/application.yaml | 11 ----------- .../src/main/resources/application.yaml | 11 ----------- 4 files changed, 37 deletions(-) diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 31627d4b7f..1b99dd97f3 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -102,10 +102,6 @@ org.codehaus.janino janino - - com.github.ben-manes.caffeine - caffeine - org.apache.hbase.thirdparty diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index a85eadb59f..c2d8f5e787 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -22,17 +22,6 @@ spring: jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats datasource: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler diff --git a/dolphinscheduler-master/src/test/resources/application.yaml b/dolphinscheduler-master/src/test/resources/application.yaml index 0dbe490af3..f4827d4b3c 100644 --- a/dolphinscheduler-master/src/test/resources/application.yaml +++ b/dolphinscheduler-master/src/test/resources/application.yaml @@ -20,17 +20,6 @@ spring: jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats datasource: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index d26ea5a4e0..654113471d 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -23,17 +23,6 @@ spring: date-format: "yyyy-MM-dd HH:mm:ss" banner: charset: UTF-8 - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats sql: init: schema-locations: classpath:sql/dolphinscheduler_h2.sql From e5e77492518a198877171801c1cf484b86f852e3 Mon Sep 17 00:00:00 2001 From: BaiJv Date: Fri, 12 Apr 2024 10:06:32 +0800 Subject: [PATCH 03/41] [Improvement] Abnormal characters check (#15824) * abnormal characters check * add test case * remove error log * fix code style * fix import --- .../service/impl/ResourcesServiceImpl.java | 5 +++++ .../api/utils/CheckUtils.java | 10 ++++++++++ .../api/utils/CheckUtilsTest.java | 20 +++++++++++++++++++ .../common/constants/Constants.java | 5 +++++ 4 files changed, 40 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 6a15da17a8..1c039cdfbd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkFilePath; import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS; import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; @@ -1290,6 +1291,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { return; } + // abnormal characters check + if (!checkFilePath(fullName)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH); + } // Avoid returning to the parent directory if (fullName.contains("../")) { throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 8b166a16dd..b394d4956c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -158,4 +158,14 @@ public class CheckUtils { return pattern.matcher(str).matches(); } + + /** + * regex FilePath check,only use a to z, A to Z, 0 to 9, and _./- + * + * @param str input string + * @return true if regex pattern is right, otherwise return false + */ + public static boolean checkFilePath(String str) { + return regexChecks(str, Constants.REGEX_FILE_PATH); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index bca8a69a16..da5ea88c83 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -92,4 +92,24 @@ public class CheckUtilsTest { Assertions.assertTrue(CheckUtils.checkPhone("17362537263")); } + /** + * check file path + */ + @Test + public void testCheckFilePath() { + // true + Assertions.assertTrue(CheckUtils.checkFilePath("/")); + Assertions.assertTrue(CheckUtils.checkFilePath("xx/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/xx")); + Assertions.assertTrue(CheckUtils.checkFilePath("14567134578654")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/1531531..13513/153135..")); + // false + Assertions.assertFalse(CheckUtils.checkFilePath(null)); + Assertions.assertFalse(CheckUtils.checkFilePath("file://xxx/ss")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss;/dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss && /dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss || /dasd/123")); + } + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 054a9410d5..19e1a1fabb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -252,6 +252,11 @@ public final class Constants { */ public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,39}$"); + /** + * file path regex + */ + public static final Pattern REGEX_FILE_PATH = Pattern.compile("^[a-zA-Z0-9_./-]+$"); + /** * read permission */ From 08ac1322864edf42903c7c03942fcad62c37da35 Mon Sep 17 00:00:00 2001 From: BaiJv Date: Fri, 12 Apr 2024 14:28:13 +0800 Subject: [PATCH 04/41] [Improvement] Modify python-gateway: enabled default to false. (#15825) * remove python-gateway:auth-token,modify python gateway: enabled default to false. * reset token --- dolphinscheduler-api/src/main/resources/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index c93fd99a59..61f9c8a592 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -149,8 +149,8 @@ api: #tenant1: 11 #tenant2: 20 python-gateway: - # Weather enable python gateway server or not. The default value is true. - enabled: true + # Weather enable python gateway server or not. The default value is false. + enabled: false # Authentication token for connection from python api to python gateway server. Should be changed the default value # when you deploy in public network. auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc From efbffacfbecd3632388e69ed400e3433ecc35f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Mon, 15 Apr 2024 11:29:11 +0800 Subject: [PATCH 05/41] delete useless code (#15844) --- .../dolphinscheduler/dao/mapper/UdfFuncMapper.java | 7 ------- .../dolphinscheduler/dao/mapper/UdfFuncMapper.xml | 12 ------------ .../dao/mapper/UdfFuncMapperTest.java | 13 ------------- 3 files changed, 32 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 6bc8049c7d..abc12f9aa1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -109,13 +109,6 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId, @Param("resourceIds") int[] resourceIds); - /** - * batch update udf func - * @param udfFuncList udf list - * @return update num - */ - int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); - /** * listAuthorizedUdfByUserId * @param userId diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index 24cb8de956..84f9fd9708 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -171,18 +171,6 @@ - - - update t_ds_udfs - - resource_name=#{udf.resourceName}, - update_time=#{udf.updateTime} - - - id=#{udf.id} - - - @@ -38,30 +38,25 @@ and log.time > #{startDate} and log.time #{endDate} - - and log.resource_type in - - #{i} + + and log.model_type in + + #{model_type} - - and log.operation in - - #{j} + + and log.operation_type in + + #{operation_type} - and u.user_name = #{userName} + and u.user_name like concat ('%', #{userName}, '%') + + + and log.model_name like concat ('%', #{modelName}, '%') order by log.time desc - diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 3c7b8805a0..4187b93a1c 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -2020,10 +2020,14 @@ CREATE TABLE t_ds_audit_log ( id int(11) NOT NULL AUTO_INCREMENT, user_id int(11) NOT NULL, - resource_type int(11) NOT NULL, - operation int(11) NOT NULL, + model_id bigint(20) NOT NULL, + model_name varchar(255) NOT NULL, + model_type varchar(255) NOT NULL, + operation_type varchar(255) NOT NULL, + description varchar(255) NOT NULL, + latency int(11) NOT NULL, + detail varchar(255) DEFAULT NULL, time timestamp NULL DEFAULT CURRENT_TIMESTAMP, - resource_id int(11) NOT NULL, PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 57401291ed..f3d01c14b5 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -2009,10 +2009,14 @@ DROP TABLE IF EXISTS `t_ds_audit_log`; CREATE TABLE `t_ds_audit_log` ( `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT'key', `user_id` int(11) NOT NULL COMMENT 'user id', - `resource_type` int(11) NOT NULL COMMENT 'resource type', - `operation` int(11) NOT NULL COMMENT 'operation', - `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', - `resource_id` int(11) NULL DEFAULT NULL COMMENT 'resource id', + `model_id` bigint(20) DEFAULT NULL COMMENT 'model id', + `model_name` varchar(100) DEFAULT NULL COMMENT 'model name', + `model_type` varchar(100) NOT NULL COMMENT 'model type', + `operation_type` varchar(100) NOT NULL COMMENT 'operation type', + `description` varchar(100) DEFAULT NULL COMMENT 'api description', + `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', + `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', + `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT= 1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 8313542ab8..5dd804c0c7 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -1993,11 +1993,15 @@ CREATE TABLE t_ds_task_group ( DROP TABLE IF EXISTS t_ds_audit_log; CREATE TABLE t_ds_audit_log ( id serial NOT NULL, - user_id int NOT NULL, - resource_type int NOT NULL, - operation int NOT NULL, - time timestamp DEFAULT NULL , - resource_id int NOT NULL, + user_id int NOT NULL, + model_id bigint NOT NULL, + model_name VARCHAR(255) NOT NULL, + model_type VARCHAR(255) NOT NULL, + operation_type VARCHAR(255) NOT NULL, + description VARCHAR(255) NOT NULL, + latency int NOT NULL, + detail VARCHAR(255) DEFAULT NULL, + time timestamp DEFAULT NULL , PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index 5ee7453d46..b5e5b31d11 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -25,4 +25,32 @@ CREATE TABLE `t_ds_relation_project_worker_group` ( UNIQUE KEY unique_project_worker_group(project_code,worker_group) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; -ALTER TABLE t_ds_project_parameter ADD `operator` int(11) DEFAULT NULL COMMENT 'operator user id'; \ No newline at end of file +ALTER TABLE t_ds_project_parameter ADD `operator` int(11) DEFAULT NULL COMMENT 'operator user id'; + +-- modify_data_t_ds_audit_log_input_entry behavior change +--DROP PROCEDURE if EXISTS modify_data_t_ds_audit_log_input_entry; +DROP PROCEDURE if EXISTS modify_data_t_ds_audit_log_input_entry; +delimiter d// +CREATE PROCEDURE modify_data_t_ds_audit_log_input_entry() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_audit_log' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='resource_type') + THEN +ALTER TABLE `t_ds_audit_log` +drop resource_type, drop operation, drop resource_id, + add `model_id` bigint(20) DEFAULT NULL COMMENT 'model id', + add `model_name` varchar(100) DEFAULT NULL COMMENT 'model name', + add `model_type` varchar(100) NOT NULL COMMENT 'model type', + add `operation_type` varchar(100) NOT NULL COMMENT 'operation type', + add `description` varchar(100) DEFAULT NULL COMMENT 'api description', + add `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', + add `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', + MODIFY COLUMN `time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; +END IF; +END; +d// +delimiter ; +CALL modify_data_t_ds_audit_log_input_entry; +DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 90646557c0..22e1c599de 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -29,4 +29,30 @@ DROP SEQUENCE IF EXISTS t_ds_relation_project_worker_group_sequence; CREATE SEQUENCE t_ds_relation_project_worker_group_sequence; ALTER TABLE t_ds_relation_project_worker_group ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_project_worker_group_sequence'); -ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS operator int; \ No newline at end of file +ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS operator int; + +-- modify_data_t_ds_audit_log_input_entry +delimiter d// +CREATE OR REPLACE FUNCTION modify_data_t_ds_audit_log_input_entry() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 't_ds_audit_log' + AND column_name = 'resource_type') + THEN +ALTER TABLE t_ds_audit_log +drop resource_type, drop operation, drop resource_id, + add model_id bigint NOT NULL, + add model_name VARCHAR(255) NOT NULL, + add model_type VARCHAR(255) NOT NULL, + add operation_type VARCHAR(255) NOT NULL, + add description VARCHAR(255) NOT NULL, + add latency int NOT NULL, + add detail VARCHAR(255) DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// + +select modify_data_t_ds_audit_log_input_entry(); +DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index cc0532032d..023b559e79 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -17,12 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; +import org.apache.dolphinscheduler.common.enums.AuditModelType; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.AuditLog; import org.apache.dolphinscheduler.dao.entity.Project; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,6 +33,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.Lists; public class AuditLogMapperTest extends BaseDaoTest { @@ -39,13 +43,17 @@ public class AuditLogMapperTest extends BaseDaoTest { @Autowired private ProjectMapper projectMapper; - private void insertOne(AuditResourceType resourceType) { + private void insertOne(AuditModelType objectType) { AuditLog auditLog = new AuditLog(); auditLog.setUserId(1); + auditLog.setModelName("name"); + auditLog.setDetail("detail"); + auditLog.setLatency(1L); auditLog.setTime(new Date()); - auditLog.setResourceType(resourceType.getCode()); - auditLog.setOperation(0); - auditLog.setResourceId(0); + auditLog.setModelType(objectType.getName()); + auditLog.setOperationType(AuditOperationType.CREATE.getName()); + auditLog.setModelId(1L); + auditLog.setDescription("description"); logMapper.insert(auditLog); } @@ -65,25 +73,14 @@ public class AuditLogMapperTest extends BaseDaoTest { */ @Test public void testQueryAuditLog() { - insertOne(AuditResourceType.USER_MODULE); - insertOne(AuditResourceType.PROJECT_MODULE); + insertOne(AuditModelType.USER); + insertOne(AuditModelType.PROJECT); Page page = new Page<>(1, 3); - int[] resourceType = new int[0]; - int[] operationType = new int[0]; + List objectTypeList = new ArrayList<>(); + List operationTypeList = Lists.newArrayList(AuditOperationType.CREATE.getName()); - IPage logIPage = logMapper.queryAuditLog(page, resourceType, operationType, "", null, null); + IPage logIPage = + logMapper.queryAuditLog(page, objectTypeList, operationTypeList, "", "", null, null); Assertions.assertNotEquals(0, logIPage.getTotal()); } - - @Test - public void testQueryResourceNameByType() { - String resourceNameByUser = logMapper.queryResourceNameByType(AuditResourceType.USER_MODULE.getMsg(), 1); - Assertions.assertEquals("admin", resourceNameByUser); - Project project = insertProject(); - String resourceNameByProject = - logMapper.queryResourceNameByType(AuditResourceType.PROJECT_MODULE.getMsg(), project.getId()); - Assertions.assertEquals(project.getName(), resourceNameByProject); - int delete = projectMapper.deleteById(project.getId()); - Assertions.assertEquals(delete, 1); - } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 2b3dbbebbb..40191f6aa3 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -1774,6 +1774,7 @@ public class ProcessServiceImpl implements ProcessService { if (Boolean.TRUE.equals(syncDefine)) { if (processDefinition.getId() == null) { result = processDefineMapper.insert(processDefinitionLog); + processDefinition.setId(processDefinitionLog.getId()); } else { processDefinitionLog.setId(processDefinition.getId()); result = processDefineMapper.updateById(processDefinitionLog); diff --git a/dolphinscheduler-ui/src/locales/en_US/monitor.ts b/dolphinscheduler-ui/src/locales/en_US/monitor.ts index 27bd82ce70..9a7550177a 100644 --- a/dolphinscheduler-ui/src/locales/en_US/monitor.ts +++ b/dolphinscheduler-ui/src/locales/en_US/monitor.ts @@ -60,9 +60,11 @@ export default { }, audit_log: { user_name: 'User Name', - resource_type: 'Resource Type', - project_name: 'Project Name', operation_type: 'Operation Type', + model_type: 'Model Type', + model_name: 'Model Name', + latency: 'Latency', + description: 'Description', create_time: 'Create Time', start_time: 'Start Time', end_time: 'End Time', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts index eca2fe7a20..dbae52fe07 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts @@ -58,9 +58,11 @@ export default { }, audit_log: { user_name: '用户名称', - resource_type: '资源类型', - project_name: '项目名称', operation_type: '操作类型', + model_type: '模型类型', + model_name: '模型名称', + latency: '耗时', + description: '描述', create_time: '创建时间', start_time: '开始时间', end_time: '结束时间', diff --git a/dolphinscheduler-ui/src/service/modules/audit/index.ts b/dolphinscheduler-ui/src/service/modules/audit/index.ts index fd436ccbb2..a436fcdbaf 100644 --- a/dolphinscheduler-ui/src/service/modules/audit/index.ts +++ b/dolphinscheduler-ui/src/service/modules/audit/index.ts @@ -25,3 +25,17 @@ export function queryAuditLogListPaging(params: AuditListReq): any { params }) } + +export function queryAuditModelType(): any { + return axios({ + url: '/projects/audit/audit-log-model-type', + method: 'get' + }) +} + +export function queryAuditLogOperationType(): any { + return axios({ + url: '/projects/audit/audit-log-operation-type', + method: 'get' + }) +} diff --git a/dolphinscheduler-ui/src/service/modules/audit/types.ts b/dolphinscheduler-ui/src/service/modules/audit/types.ts index 030cfc8037..e25efada31 100644 --- a/dolphinscheduler-ui/src/service/modules/audit/types.ts +++ b/dolphinscheduler-ui/src/service/modules/audit/types.ts @@ -45,4 +45,20 @@ interface AuditListRes { start: number } -export { AuditListReq, AuditListRes } +interface AuditModelTypeItem { + code: number + name: string + child: AuditModelTypeItem[] | null +} + +interface AuditOperationTypeItem { + code: number + name: string +} + +export { + AuditListReq, + AuditListRes, + AuditModelTypeItem, + AuditOperationTypeItem +} diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx index 64ae7beff0..17de81fb51 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx @@ -30,7 +30,8 @@ import { NButton, NIcon, NDataTable, - NPagination + NPagination, + NCascader } from 'naive-ui' import { SearchOutlined } from '@vicons/antd' import { useTable } from './use-table' @@ -40,15 +41,23 @@ import Card from '@/components/card' const AuditLog = defineComponent({ name: 'audit-log', setup() { - const { t, variables, getTableData, createColumns } = useTable() + const { + t, + variables, + getTableData, + createColumns, + getModelTypeData, + getOperationTypeData + } = useTable() const requestTableData = () => { getTableData({ pageSize: variables.pageSize, pageNo: variables.page, - resourceType: variables.resourceType, + modelType: variables.modelType, operationType: variables.operationType, userName: variables.userName, + modelName: variables.modelName, datePickerRange: variables.datePickerRange }) } @@ -67,6 +76,8 @@ const AuditLog = defineComponent({ onMounted(() => { createColumns(variables) + getModelTypeData() + getOperationTypeData() requestTableData() }) @@ -97,36 +108,41 @@ const AuditLog = defineComponent({ placeholder={t('monitor.audit_log.user_name')} clearable /> + + - + diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts index 87e5215030..e0a17bbc49 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts @@ -18,22 +18,40 @@ import { useI18n } from 'vue-i18n' import { reactive, ref } from 'vue' import { useAsyncState } from '@vueuse/core' -import { queryAuditLogListPaging } from '@/service/modules/audit' +import { + queryAuditLogListPaging, + queryAuditModelType, + queryAuditLogOperationType +} from '@/service/modules/audit' import { format } from 'date-fns' import { parseTime } from '@/common/common' -import type { AuditListRes } from '@/service/modules/audit/types' +import type { + AuditListRes, + AuditModelTypeItem, + AuditOperationTypeItem +} from '@/service/modules/audit/types' +import { + COLUMN_WIDTH_CONFIG, + calculateTableWidth, + DefaultTableWidth +} from '@/common/column-width-config' +import { sortBy } from 'lodash' export function useTable() { const { t } = useI18n() const variables = reactive({ columns: [], + tableWidth: DefaultTableWidth, tableData: [], page: ref(1), pageSize: ref(10), - resourceType: ref(null), + modelType: ref(null), operationType: ref(null), + ModelTypeData: [], + OperationTypeData: [], userName: ref(null), + modelName: ref(null), datePickerRange: ref(null), totalPage: ref(1), loadingRef: ref(false) @@ -44,29 +62,70 @@ export function useTable() { { title: '#', key: 'index', - render: (row: any, index: number) => index + 1 + render: (row: any, index: number) => index + 1, + ...COLUMN_WIDTH_CONFIG['index'] }, { title: t('monitor.audit_log.user_name'), - key: 'userName' + key: 'userName', + ...COLUMN_WIDTH_CONFIG['userName'] }, { - title: t('monitor.audit_log.resource_type'), - key: 'resource' + title: t('monitor.audit_log.model_type'), + key: 'modelType', + ...COLUMN_WIDTH_CONFIG['type'] }, { - title: t('monitor.audit_log.project_name'), - key: 'resourceName' + title: t('monitor.audit_log.model_name'), + key: 'modelName', + ...COLUMN_WIDTH_CONFIG['name'] }, { title: t('monitor.audit_log.operation_type'), - key: 'operation' + key: 'operation', + ...COLUMN_WIDTH_CONFIG['type'] + }, + + { + title: t('monitor.audit_log.description'), + key: 'description', + ...COLUMN_WIDTH_CONFIG['note'] + }, + { + title: t('monitor.audit_log.latency') + ' (ms)', + key: 'latency', + ...COLUMN_WIDTH_CONFIG['times'] }, { title: t('monitor.audit_log.create_time'), - key: 'time' + key: 'time', + ...COLUMN_WIDTH_CONFIG['time'] } ] + + if (variables.tableWidth) { + variables.tableWidth = calculateTableWidth(variables.columns) + } + } + + const getModelTypeData = async () => { + try { + variables.ModelTypeData = await queryAuditModelType().then( + (res: AuditModelTypeItem[]) => res || [] + ) + } catch { + variables.ModelTypeData = [] + } + } + + const getOperationTypeData = async () => { + try { + variables.OperationTypeData = await queryAuditLogOperationType().then( + (res: AuditOperationTypeItem[]) => sortBy(res, 'name') + ) + } catch { + variables.OperationTypeData = [] + } } const getTableData = (params: any) => { @@ -75,9 +134,10 @@ export function useTable() { const data = { pageSize: params.pageSize, pageNo: params.pageNo, - resourceType: params.resourceType, - operationType: params.operationType, + modelTypes: params.modelType, + operationTypes: params.operationType, userName: params.userName, + modelName: params.modelName, startDate: params.datePickerRange ? format(parseTime(params.datePickerRange[0]), 'yyyy-MM-dd HH:mm:ss') : '', @@ -106,6 +166,8 @@ export function useTable() { t, variables, getTableData, - createColumns + createColumns, + getModelTypeData: getModelTypeData, + getOperationTypeData } } From 7894ebb3506d4281c27715f684e9fb5701bf5351 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 15 Apr 2024 14:15:44 +0800 Subject: [PATCH 07/41] [TEST] increase coverage of environment service (#15840) * [TEST] increase coverage of environment service * spotless apply --- .../service/impl/EnvironmentServiceTest.java | 88 ++++++++++++++++++- .../api/utils/ServiceTestUtil.java | 30 +++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java index 14e1ed956b..b226295e0e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServi import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_UPDATE; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -29,9 +30,11 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.ServiceTestUtil; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.User; @@ -42,6 +45,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -53,6 +57,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -91,6 +96,9 @@ public class EnvironmentServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; + @Mock + private CodeGenerateUtils codeGenerateUtils; + public static final String testUserName = "environmentServerTest"; public static final String environmentName = "Env1"; @@ -121,9 +129,24 @@ public class EnvironmentServiceTest { when(environmentMapper.insert(any(Environment.class))).thenReturn(1); when(relationMapper.insert(any(EnvironmentWorkerGroupRelation.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); assertDoesNotThrow( () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + when(environmentMapper.insert(any(Environment.class))).thenReturn(-1); + assertThrowsServiceException(Status.CREATE_ENVIRONMENT_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.getInstance()).thenReturn(codeGenerateUtils); + when(codeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + } } @Test @@ -156,25 +179,54 @@ public class EnvironmentServiceTest { assertThrowsServiceException(Status.ENVIRONMENT_NAME_EXISTS, () -> environmentService .updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), getDesc(), workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(-1); + assertThrowsServiceException(Status.UPDATE_ENVIRONMENT_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", workerGroups)); + + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); + + when(relationMapper.queryByEnvironmentCode(any())) + .thenReturn(Collections.singletonList(getEnvironmentWorkerGroup())); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); } @Test public void testQueryAllEnvironmentList() { + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, + 1, environmentServiceLogger)).thenReturn(Collections.emptySet()); + Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + assertEquals(0, ((List) result.get(Constants.DATA_LIST)).size()); + Set ids = new HashSet<>(); ids.add(1); when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, 1, environmentServiceLogger)).thenReturn(ids); when(environmentMapper.selectBatchIds(ids)).thenReturn(Lists.newArrayList(getEnvironment())); - Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List list = (List) (result.get(Constants.DATA_LIST)); Assertions.assertEquals(1, list.size()); + + when(environmentMapper.selectBatchIds(ids)).thenReturn(Collections.emptyList()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); + Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + list = (List) (result.get(Constants.DATA_LIST)); + Assertions.assertEquals(0, list.size()); } @Test @@ -186,9 +238,27 @@ public class EnvironmentServiceTest { .thenReturn(page); Result result = environmentService.queryEnvironmentListPaging(getAdminUser(), 1, 10, environmentName); - logger.info(result.toString()); PageInfo pageInfo = (PageInfo) result.getData(); Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); + + assertDoesNotThrow( + () -> environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName)); + + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition( + AuthorizationType.ENVIRONMENT, + 1, + environmentServiceLogger)).thenReturn(Collections.singleton(10)); + when(environmentMapper.queryEnvironmentListPagingByIds(any(Page.class), any(List.class), any(String.class))) + .thenReturn(page); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(1, ((PageInfo) result.getData()).getTotalList().size()); + + page.setRecords(Collections.emptyList()); + page.setTotal(0); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(0, ((PageInfo) result.getData()).getTotalList().size()); } @Test @@ -239,6 +309,10 @@ public class EnvironmentServiceTest { result = environmentService.deleteEnvironmentByCode(loginUser, 1L); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + when(environmentMapper.deleteByCode(1L)).thenReturn(-1); + result = environmentService.deleteEnvironmentByCode(loginUser, 1L); + Assertions.assertEquals(Status.DELETE_ENVIRONMENT_ERROR, result.get(Constants.STATUS)); } @Test @@ -251,6 +325,9 @@ public class EnvironmentServiceTest { result = environmentService.verifyEnvironment(environmentName); logger.info(result.toString()); Assertions.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + + when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(null); + assertDoesNotThrow(() -> environmentService.verifyEnvironment(environmentName)); } private Environment getEnvironment() { @@ -264,6 +341,13 @@ public class EnvironmentServiceTest { return environment; } + private EnvironmentWorkerGroupRelation getEnvironmentWorkerGroup() { + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(1L); + relation.setWorkerGroup("new_worker_group"); + return relation; + } + /** * create an environment description */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java new file mode 100644 index 0000000000..0dc71841b4 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import java.nio.charset.StandardCharsets; +import java.util.Random; + +public class ServiceTestUtil { + + public static String randomStringWithLengthN(int n) { + byte[] bitArray = new byte[n]; + new Random().nextBytes(bitArray); + return new String(bitArray, StandardCharsets.UTF_8); + } +} From 5ad7b1509f79c52b24d11f50c1d0172287dcd968 Mon Sep 17 00:00:00 2001 From: XinXing <49302071+xinxingi@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:01:28 +0800 Subject: [PATCH 08/41] =?UTF-8?q?[Fix-15787]=20Reuse=20code=20and=20solve?= =?UTF-8?q?=20the=20problem=20of=20complex=20SQL=20parsing=20exceptions=20?= =?UTF-8?q?in=E2=80=A6=20(#15833)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reuse code and solve the problem of complex SQL parsing exceptions in druid, corresponding to issue #15787 * Code Format * Enhanced adaptability to SQL formatting --- .../oracle/param/OracleDataSourceProcessor.java | 11 ++++++----- .../oracle/param/OracleDataSourceProcessorTest.java | 13 ++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java index 89b872d7f5..5f24b8cc19 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java @@ -41,8 +41,7 @@ import java.util.stream.Collectors; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; -import com.alibaba.druid.sql.parser.SQLParserFeature; -import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.alibaba.druid.sql.parser.SQLParserUtils; import com.google.auto.service.AutoService; @AutoService(DataSourceProcessor.class) @@ -149,9 +148,11 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - SQLStatementParser parser = new OracleStatementParser(sql, SQLParserFeature.KeepComments); - List statementList = parser.parseStatementList(); - return statementList.stream().map(SQLStatement::toString).collect(Collectors.toList()); + if (sql.toUpperCase().contains("BEGIN") && sql.toUpperCase().contains("END")) { + return new OracleStatementParser(sql).parseStatementList().stream().map(SQLStatement::toString) + .collect(Collectors.toList()); + } + return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.oracle); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java index 57e316ed63..2f9133463d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java @@ -155,13 +155,16 @@ public class OracleDataSourceProcessorTest { @Test void splitAndRemoveComment_MultipleSql() { - String plSql = "select * from test;select * from test2;"; + String plSql = + "select a,a-a as b from (select 1 as a,2 as b from dual) union all select 1 as a,2 as b from dual;select * from dual; -- this comment"; List sqls = oracleDatasourceProcessor.splitAndRemoveComment(plSql); // We will not split the plsql Assertions.assertEquals(2, sqls.size()); - Assertions.assertEquals("SELECT *\n" + - "FROM test;", sqls.get(0)); - Assertions.assertEquals("SELECT *\n" + - "FROM test2;", sqls.get(1)); + System.out.println(sqls.get(0)); + System.out.println(sqls.get(1)); + Assertions.assertEquals( + "select a,a-a as b from (select 1 as a,2 as b from dual) union all select 1 as a,2 as b from dual", + sqls.get(0)); + Assertions.assertEquals("select * from dual", sqls.get(1)); } } From 6844c659bf44f7f027adbe7a64723a1ca0f1cd89 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 15 Apr 2024 16:14:32 +0800 Subject: [PATCH 09/41] Fix ErrorCommand loss some fields in Command (#15847) --- .../dao/entity/ErrorCommand.java | 17 +++- .../dao/entity/ErrorCommandTest.java | 82 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index aa4d3d4782..d52984b61f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -52,6 +52,10 @@ public class ErrorCommand { */ private long processDefinitionCode; + private int processDefinitionVersion; + + private int processInstanceId; + /** * executor id */ @@ -73,7 +77,7 @@ public class ErrorCommand { private FailureStrategy failureStrategy; /** - * warning type + * warning type */ private WarningType warningType; @@ -135,21 +139,26 @@ public class ErrorCommand { public ErrorCommand() { } + public ErrorCommand(Command command, String message) { this.id = command.getId(); this.commandType = command.getCommandType(); this.executorId = command.getExecutorId(); this.processDefinitionCode = command.getProcessDefinitionCode(); + this.processDefinitionVersion = command.getProcessDefinitionVersion(); + this.processInstanceId = command.getProcessInstanceId(); this.commandParam = command.getCommandParam(); + this.taskDependType = command.getTaskDependType(); + this.failureStrategy = command.getFailureStrategy(); this.warningType = command.getWarningType(); this.warningGroupId = command.getWarningGroupId(); this.scheduleTime = command.getScheduleTime(); - this.taskDependType = command.getTaskDependType(); - this.failureStrategy = command.getFailureStrategy(); this.startTime = command.getStartTime(); this.updateTime = command.getUpdateTime(); - this.environmentCode = command.getEnvironmentCode(); this.processInstancePriority = command.getProcessInstancePriority(); + this.workerGroup = command.getWorkerGroup(); + this.tenantCode = command.getTenantCode(); + this.environmentCode = command.getEnvironmentCode(); this.message = message; this.dryRun = command.getDryRun(); this.testFlag = command.getTestFlag(); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java new file mode 100644 index 0000000000..057d6578e6 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; + +import java.util.Date; + +import org.junit.jupiter.api.Test; + +class ErrorCommandTest { + + @Test + void testConstructor() { + Command command = new Command(); + command.setId(1); + command.setCommandType(CommandType.PAUSE); + command.setExecutorId(1); + command.setProcessDefinitionCode(123); + command.setProcessDefinitionVersion(1); + command.setProcessInstanceId(1); + command.setCommandParam("param"); + command.setTaskDependType(TaskDependType.TASK_POST); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(new Date()); + command.setStartTime(new Date()); + command.setUpdateTime(new Date()); + command.setProcessInstancePriority(Priority.HIGHEST); + command.setWorkerGroup("default"); + command.setTenantCode("root"); + command.setEnvironmentCode(1L); + command.setDryRun(1); + command.setTestFlag(Flag.NO.getCode()); + + ErrorCommand errorCommand = new ErrorCommand(command, "test"); + assertEquals(command.getCommandType(), errorCommand.getCommandType()); + assertEquals(command.getExecutorId(), errorCommand.getExecutorId()); + assertEquals(command.getProcessDefinitionCode(), errorCommand.getProcessDefinitionCode()); + assertEquals(command.getProcessDefinitionVersion(), errorCommand.getProcessDefinitionVersion()); + assertEquals(command.getProcessInstanceId(), errorCommand.getProcessInstanceId()); + assertEquals(command.getCommandParam(), errorCommand.getCommandParam()); + assertEquals(command.getTaskDependType(), errorCommand.getTaskDependType()); + assertEquals(command.getFailureStrategy(), errorCommand.getFailureStrategy()); + assertEquals(command.getWarningType(), errorCommand.getWarningType()); + assertEquals(command.getWarningGroupId(), errorCommand.getWarningGroupId()); + assertEquals(command.getScheduleTime(), errorCommand.getScheduleTime()); + assertEquals(command.getStartTime(), errorCommand.getStartTime()); + assertEquals(command.getUpdateTime(), errorCommand.getUpdateTime()); + assertEquals(command.getProcessInstancePriority(), errorCommand.getProcessInstancePriority()); + assertEquals(command.getWorkerGroup(), errorCommand.getWorkerGroup()); + assertEquals(command.getTenantCode(), errorCommand.getTenantCode()); + assertEquals(command.getEnvironmentCode(), errorCommand.getEnvironmentCode()); + assertEquals(command.getDryRun(), errorCommand.getDryRun()); + assertEquals(command.getTestFlag(), errorCommand.getTestFlag()); + assertEquals("test", errorCommand.getMessage()); + } + +} From f99d3f1ed37f1f36a51cfa2063d298f1bed5de49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Mon, 15 Apr 2024 17:01:06 +0800 Subject: [PATCH 10/41] [Improvement][Audit] Change time to create_time (#15846) * change time to create_time * update * update --- .../api/audit/OperatorUtils.java | 2 +- .../dolphinscheduler/api/dto/AuditDto.java | 2 +- .../api/service/impl/AuditServiceImpl.java | 2 +- .../dolphinscheduler/dao/entity/AuditLog.java | 2 +- .../dao/mapper/AuditLogMapper.xml | 8 ++++---- .../resources/sql/dolphinscheduler_h2.sql | 2 +- .../resources/sql/dolphinscheduler_mysql.sql | 2 +- .../sql/dolphinscheduler_postgresql.sql | 2 +- .../mysql/dolphinscheduler_ddl.sql | 2 +- .../postgresql/dolphinscheduler_ddl.sql | 19 ++++++++++--------- .../dao/mapper/AuditLogMapperTest.java | 2 +- .../monitor/statistics/audit-log/use-table.ts | 2 +- 12 files changed, 24 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index e8b6b1254c..a0233c7e93 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -65,7 +65,7 @@ public class OperatorUtils { auditLog.setModelType(auditType.getAuditModelType().getName()); auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); - auditLog.setTime(new Date()); + auditLog.setCreateTime(new Date()); auditLogList.add(auditLog); return auditLogList; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java index 2c4144220b..0b36325b44 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java @@ -34,7 +34,7 @@ public class AuditDto { private String operation; - private Date time; + private Date createTime; private String description; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index bdc7a4c573..2ec547bcb6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -129,7 +129,7 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { auditDto.setLatency(String.valueOf(auditLog.getLatency())); auditDto.setDetail(auditLog.getDetail()); auditDto.setDescription(auditLog.getDescription()); - auditDto.setTime(auditLog.getTime()); + auditDto.setCreateTime(auditLog.getCreateTime()); return auditDto; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java index 397fa722e5..7c6f76e1da 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java @@ -67,7 +67,7 @@ public class AuditLog { /** * operation time */ - private Date time; + private Date createTime; private String detail; diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml index 94430a343a..63adb57c34 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml @@ -19,10 +19,10 @@ - id, user_id, model_type, operation_type, model_id, model_name, time, detail, description, latency + id, user_id, model_type, operation_type, model_id, model_name, create_time, detail, description, latency - ${alias}.id, ${alias}.user_id, ${alias}.model_type, ${alias}.operation_type, ${alias}.model_id, ${alias}.model_name, ${alias}.time, ${alias}.detail, ${alias}.description, ${alias}.latency + ${alias}.id, ${alias}.user_id, ${alias}.model_type, ${alias}.operation_type, ${alias}.model_id, ${alias}.model_name, ${alias}.create_time, ${alias}.detail, ${alias}.description, ${alias}.latency diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 4187b93a1c..42c893bb05 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -2027,7 +2027,7 @@ CREATE TABLE t_ds_audit_log description varchar(255) NOT NULL, latency int(11) NOT NULL, detail varchar(255) DEFAULT NULL, - time timestamp NULL DEFAULT CURRENT_TIMESTAMP, + create_time timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index f3d01c14b5..e2138b67a2 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -2016,7 +2016,7 @@ CREATE TABLE `t_ds_audit_log` ( `description` varchar(100) DEFAULT NULL COMMENT 'api description', `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', - `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT= 1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 5dd804c0c7..2d224092c4 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -2001,7 +2001,7 @@ CREATE TABLE t_ds_audit_log ( description VARCHAR(255) NOT NULL, latency int NOT NULL, detail VARCHAR(255) DEFAULT NULL, - time timestamp DEFAULT NULL , + create_time timestamp DEFAULT NULL , PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index b5e5b31d11..d10ac6b710 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -47,7 +47,7 @@ drop resource_type, drop operation, drop resource_id, add `description` varchar(100) DEFAULT NULL COMMENT 'api description', add `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', add `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', - MODIFY COLUMN `time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; + CHANGE COLUMN `time` `create_time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; END IF; END; d// diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 22e1c599de..ee06588e58 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -40,15 +40,16 @@ BEGIN WHERE table_name = 't_ds_audit_log' AND column_name = 'resource_type') THEN -ALTER TABLE t_ds_audit_log -drop resource_type, drop operation, drop resource_id, - add model_id bigint NOT NULL, - add model_name VARCHAR(255) NOT NULL, - add model_type VARCHAR(255) NOT NULL, - add operation_type VARCHAR(255) NOT NULL, - add description VARCHAR(255) NOT NULL, - add latency int NOT NULL, - add detail VARCHAR(255) DEFAULT NULL; + ALTER TABLE t_ds_audit_log + drop resource_type, drop operation, drop resource_id, + add model_id bigint NOT NULL, + add model_name VARCHAR(255) NOT NULL, + add model_type VARCHAR(255) NOT NULL, + add operation_type VARCHAR(255) NOT NULL, + add description VARCHAR(255) NOT NULL, + add latency int NOT NULL, + add detail VARCHAR(255) DEFAULT NULL; + ALTER TABLE t_ds_audit_log RENAME COLUMN "time" TO "create_time"; END IF; END; $$ LANGUAGE plpgsql; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index 023b559e79..92418193b8 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -49,7 +49,7 @@ public class AuditLogMapperTest extends BaseDaoTest { auditLog.setModelName("name"); auditLog.setDetail("detail"); auditLog.setLatency(1L); - auditLog.setTime(new Date()); + auditLog.setCreateTime(new Date()); auditLog.setModelType(objectType.getName()); auditLog.setOperationType(AuditOperationType.CREATE.getName()); auditLog.setModelId(1L); diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts index e0a17bbc49..385299d019 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts @@ -98,7 +98,7 @@ export function useTable() { }, { title: t('monitor.audit_log.create_time'), - key: 'time', + key: 'createTime', ...COLUMN_WIDTH_CONFIG['time'] } ] From 27d0563fe4c020a771448830309429592c66ba9c Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 10:58:43 +0800 Subject: [PATCH 11/41] Bind processId to constructor CodeGenerator (#15848) --- .../api/python/PythonGateway.java | 6 +- .../api/service/impl/ClusterServiceImpl.java | 2 +- .../service/impl/EnvironmentServiceImpl.java | 2 +- .../api/service/impl/ExecutorServiceImpl.java | 2 +- .../service/impl/K8SNamespaceServiceImpl.java | 2 +- .../impl/ProcessDefinitionServiceImpl.java | 16 ++-- .../impl/ProjectParameterServiceImpl.java | 2 +- .../impl/ProjectPreferenceServiceImpl.java | 2 +- .../api/service/impl/ProjectServiceImpl.java | 2 +- .../impl/TaskDefinitionServiceImpl.java | 6 +- .../service/impl/EnvironmentServiceTest.java | 6 +- .../common/utils/CodeGenerateUtils.java | 95 ++++++++++--------- .../common/utils/CodeGenerateUtilsTest.java | 55 +++++++++-- .../service/process/ProcessServiceImpl.java | 2 +- .../datasource/dao/ProcessDefinitionDao.java | 2 +- .../tools/datasource/dao/ProjectDao.java | 2 +- .../v200/V200DolphinSchedulerUpgrader.java | 2 +- .../tools/demo/ProcessDefinitionDemo.java | 16 ++-- 18 files changed, 132 insertions(+), 90 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 1e4e1f5aa0..762ba576b0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -184,7 +184,7 @@ public class PythonGateway { Map result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value if (project == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -194,7 +194,7 @@ public class PythonGateway { // In the case project exists, but current workflow still not created, we should also return the init // version of it if (processDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -202,7 +202,7 @@ public class PythonGateway { TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); if (taskDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); } else { result.put("code", taskDefinition.getCode()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index a5ceb92abc..bf4e3ac4d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -96,7 +96,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic cluster.setOperator(loginUser.getId()); cluster.setCreateTime(new Date()); cluster.setUpdateTime(new Date()); - cluster.setCode(CodeGenerateUtils.getInstance().genCode()); + cluster.setCode(CodeGenerateUtils.genCode()); if (clusterMapper.insert(cluster) > 0) { return cluster.getCode(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index ade9aea483..2eb8f84810 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -123,7 +123,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme env.setUpdateTime(new Date()); long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); env.setCode(code); } catch (CodeGenerateException e) { log.error("Generate environment code error.", e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 7ab6102ff8..5b576ce95a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -258,7 +258,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ checkScheduleTimeNumExceed(commandType, cronTime); checkMasterExists(); - long triggerCode = CodeGenerateUtils.getInstance().genCode(); + long triggerCode = CodeGenerateUtils.genCode(); /** * create command diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java index 54894a5ca7..7543616f31 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java @@ -141,7 +141,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); cluster.setCode(code); } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("Generate cluster code error.", e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index ed6b2d27bb..2bf84a0bcc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -299,7 +299,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro List taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson); List taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs); - long processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + long processDefinitionCode = CodeGenerateUtils.genCode(); ProcessDefinition processDefinition = new ProcessDefinition(projectCode, name, processDefinitionCode, description, globalParams, locations, timeout, loginUser.getId()); @@ -360,7 +360,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro long processDefinitionCode; try { - processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + processDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -1233,7 +1233,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // build process definition processDefinition = new ProcessDefinition(projectCode, processDefinitionName, - CodeGenerateUtils.getInstance().genCode(), + CodeGenerateUtils.genCode(), "", "[]", null, 0, loginUser.getId()); @@ -1388,7 +1388,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro sqlParameters.setSqlType(SqlType.NON_QUERY.ordinal()); sqlParameters.setLocalParams(Collections.emptyList()); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); taskDefinition.setTaskType(TASK_TYPE_SQL); taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryInterval(0); @@ -1433,7 +1433,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processDefinition.setProjectCode(projectCode); processDefinition.setUserId(loginUser.getId()); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error( "Save process definition error because generate process definition code error, projectCode:{}.", @@ -1456,7 +1456,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperateTime(now); try { - long code = CodeGenerateUtils.getInstance().genCode(); + long code = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), code); taskDefinitionLog.setCode(code); } catch (CodeGenerateException e) { @@ -2074,7 +2074,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro Map taskCodeMap = new HashMap<>(); for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { try { - long taskCode = CodeGenerateUtils.getInstance().genCode(); + long taskCode = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), taskCode); taskDefinitionLog.setCode(taskCode); } catch (CodeGenerateException e) { @@ -2097,7 +2097,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } final long oldProcessDefinitionCode = processDefinition.getCode(); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error("Generate process definition code error, projectCode:{}.", targetProjectCode, e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e30375e809..e0011096e4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -97,7 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj .builder() .paramName(projectParameterName) .paramValue(projectParameterValue) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .projectCode(projectCode) .userId(loginUser.getId()) .createTime(now) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java index dcbd3b7456..6274d290d5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java @@ -76,7 +76,7 @@ public class ProjectPreferenceServiceImpl extends BaseServiceImpl projectPreference.setProjectCode(projectCode); projectPreference.setPreferences(preferences); projectPreference.setUserId(loginUser.getId()); - projectPreference.setCode(CodeGenerateUtils.getInstance().genCode()); + projectPreference.setCode(CodeGenerateUtils.genCode()); projectPreference.setState(1); projectPreference.setCreateTime(now); projectPreference.setUpdateTime(now); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 1a00584a26..b5c329c4fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -126,7 +126,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic project = Project .builder() .name(name) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .description(desc) .userId(loginUser.getId()) .userName(loginUser.getUserName()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 4a0c3f8b68..8b01df1319 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -265,7 +265,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe long taskDefinitionCode; try { - taskDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + taskDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -338,7 +338,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe } long taskCode = taskDefinition.getCode(); if (taskCode == 0) { - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); } List processTaskRelationLogList = processTaskRelationMapper.queryByProcessCode(processDefinitionCode) @@ -1264,7 +1264,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe List taskCodes = new ArrayList<>(); try { for (int i = 0; i < genNum; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateException e) { log.error("Generate task definition code error.", e); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java index b226295e0e..b8161af13d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java @@ -96,9 +96,6 @@ public class EnvironmentServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; - @Mock - private CodeGenerateUtils codeGenerateUtils; - public static final String testUserName = "environmentServerTest"; public static final String environmentName = "Env1"; @@ -141,8 +138,7 @@ public class EnvironmentServiceTest { () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { - when(CodeGenerateUtils.getInstance()).thenReturn(codeGenerateUtils); - when(codeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java index f35523b59d..3e75264808 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java @@ -1,4 +1,6 @@ -/** Copyright 2010-2012 Twitter, Inc.*/ +/** + * Copyright 2010-2012 Twitter, Inc. + */ package org.apache.dolphinscheduler.common.utils; @@ -6,66 +8,71 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; + /** - * Rewriting based on Twitter snowflake algorithm + * Rewriting based on Twitter snowflake algorithm */ +@Slf4j public class CodeGenerateUtils { - // start timestamp - private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 - // Each machine generates 32 in the same millisecond - private static final long LOW_DIGIT_BIT = 5L; - private static final long MIDDLE_BIT = 2L; - private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); - // The displacement to the left - private static final long MIDDLE_LEFT = LOW_DIGIT_BIT; - private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT; - private final long machineHash; - private long lowDigit = 0L; - private long recordMillisecond = -1L; + private static final CodeGenerator codeGenerator; - private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); - private static final long SYSTEM_NANOTIME = System.nanoTime(); - - private CodeGenerateUtils() throws CodeGenerateException { + static { try { - this.machineHash = - Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1)); + codeGenerator = new CodeGenerator(InetAddress.getLocalHost().getHostName() + "-" + OSUtils.getProcessID()); } catch (UnknownHostException e) { throw new CodeGenerateException(e.getMessage()); } } - private static CodeGenerateUtils instance = null; - - public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException { - if (instance == null) { - instance = new CodeGenerateUtils(); - } - return instance; + public static long genCode() throws CodeGenerateException { + return codeGenerator.genCode(); } - public synchronized long genCode() throws CodeGenerateException { - long nowtMillisecond = systemMillisecond(); - if (nowtMillisecond < recordMillisecond) { - throw new CodeGenerateException("New code exception because time is set back."); + public static class CodeGenerator { + + // start timestamp + private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 + // Each machine generates 32 in the same millisecond + private static final long LOW_DIGIT_BIT = 5L; + private static final long MACHINE_BIT = 5L; + private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); + // The displacement to the left + private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MACHINE_BIT; + public final long machineHash; + private long lowDigit = 0L; + private long recordMillisecond = -1L; + + private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); + private static final long SYSTEM_NANOTIME = System.nanoTime(); + + public CodeGenerator(String appName) { + this.machineHash = Math.abs(Objects.hash(appName)) % (1 << MACHINE_BIT); } - if (nowtMillisecond == recordMillisecond) { - lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; - if (lowDigit == 0L) { - while (nowtMillisecond <= recordMillisecond) { - nowtMillisecond = systemMillisecond(); - } + + public synchronized long genCode() throws CodeGenerateException { + long nowtMillisecond = systemMillisecond(); + if (nowtMillisecond < recordMillisecond) { + throw new CodeGenerateException("New code exception because time is set back."); } - } else { - lowDigit = 0L; + if (nowtMillisecond == recordMillisecond) { + lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; + if (lowDigit == 0L) { + while (nowtMillisecond <= recordMillisecond) { + nowtMillisecond = systemMillisecond(); + } + } + } else { + lowDigit = 0L; + } + recordMillisecond = nowtMillisecond; + return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << LOW_DIGIT_BIT | lowDigit; } - recordMillisecond = nowtMillisecond; - return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit; - } - private long systemMillisecond() { - return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + private long systemMillisecond() { + return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + } } public static class CodeGenerateException extends RuntimeException { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index 3871646c95..8cd8ab8e6d 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -17,20 +17,59 @@ package org.apache.dolphinscheduler.common.utils; -import java.util.HashSet; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class CodeGenerateUtilsTest { +class CodeGenerateUtilsTest { @Test - public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { - HashSet existsCode = new HashSet<>(); - for (int i = 0; i < 100; i++) { - Long currentCode = CodeGenerateUtils.getInstance().genCode(); - Assertions.assertFalse(existsCode.contains(currentCode)); + void testNoGenerateDuplicateCode() { + int codeNum = 10000000; + List existsCode = new ArrayList<>(); + for (int i = 0; i < codeNum; i++) { + Long currentCode = CodeGenerateUtils.genCode(); existsCode.add(currentCode); } + Set existsCodeSet = new HashSet<>(existsCode); + // Disallow duplicate code + assertEquals(existsCode.size(), existsCodeSet.size()); + } + + @Test + void testNoGenerateDuplicateCodeWithDifferentAppName() throws UnknownHostException, InterruptedException { + int threadNum = 10; + int codeNum = 1000000; + + final String hostName = InetAddress.getLocalHost().getHostName(); + Map> machineCodes = new ConcurrentHashMap<>(); + CountDownLatch countDownLatch = new CountDownLatch(threadNum); + + for (int i = 0; i < threadNum; i++) { + final int c = i; + new Thread(() -> { + List codes = new ArrayList<>(codeNum); + CodeGenerateUtils.CodeGenerator codeGenerator = new CodeGenerateUtils.CodeGenerator(hostName + "-" + c); + for (int j = 0; j < codeNum; j++) { + codes.add(codeGenerator.genCode()); + } + machineCodes.put(Thread.currentThread().getName(), codes); + countDownLatch.countDown(); + }).start(); + } + countDownLatch.await(); + Set totalCodes = new HashSet<>(); + machineCodes.values().forEach(totalCodes::addAll); + assertEquals(codeNum * threadNum, totalCodes.size()); } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 40191f6aa3..028ab7651f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -1679,7 +1679,7 @@ public class ProcessServiceImpl implements ProcessService { taskDefinitionLog.setOperateTime(now); taskDefinitionLog.setOperator(operator.getId()); if (taskDefinitionLog.getCode() == 0) { - taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinitionLog.setCode(CodeGenerateUtils.genCode()); } if (taskDefinitionLog.getVersion() == 0) { // init first version diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java index 338a9d591d..877f1c9482 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java @@ -100,7 +100,7 @@ public class ProcessDefinitionDao { processDefinition.setId(rs.getInt(1)); long code = rs.getLong(2); if (code == 0L) { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); } processDefinition.setCode(code); processDefinition.setVersion(Constants.VERSION_FIRST); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java index 65466fe99a..685732337a 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java @@ -47,7 +47,7 @@ public class ProjectDao { Integer id = rs.getInt(1); long code = rs.getLong(2); if (code == 0L) { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); } projectMap.put(id, code); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java index 35a9be75dc..fd430c2b06 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java @@ -206,7 +206,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { taskDefinitionLog.setName(name); taskDefinitionLog .setWorkerGroup(task.get("workerGroup") == null ? "default" : task.get("workerGroup").asText()); - long taskCode = CodeGenerateUtils.getInstance().genCode(); + long taskCode = CodeGenerateUtils.genCode(); taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setVersion(Constants.VERSION_FIRST); taskDefinitionLog.setProjectCode(processDefinition.getProjectCode()); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java index fb1cfab637..023928575f 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java @@ -85,7 +85,7 @@ public class ProcessDefinitionDemo { project = Project .builder() .name("demo") - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .description("") .userId(loginUser.getId()) .userName(loginUser.getUserName()) @@ -167,7 +167,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 1; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -242,7 +242,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 2; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -334,7 +334,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 2; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -420,7 +420,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 4; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -537,7 +537,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 4; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -656,7 +656,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 3; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -755,7 +755,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 1; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); From 76d059810a928c3a122df3ee917cb5eedaf60063 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 16:19:57 +0800 Subject: [PATCH 12/41] Forbidden forcess success a task instance in a running workflow instance (#15855) --- .../controller/TaskInstanceController.java | 9 +- .../v2/TaskInstanceV2Controller.java | 4 +- .../api/service/TaskInstanceService.java | 6 +- .../service/impl/TaskInstanceServiceImpl.java | 64 ++++------ .../api/AssertionsHelper.java | 8 ++ .../TaskInstanceControllerTest.java | 4 +- .../v2/TaskInstanceV2ControllerTest.java | 8 +- .../api/service/TaskInstanceServiceTest.java | 119 +++++++++--------- .../service/process/ProcessService.java | 2 +- .../service/process/ProcessServiceImpl.java | 13 +- 10 files changed, 116 insertions(+), 121 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 1865dfee5d..e0055595c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -153,10 +153,11 @@ public class TaskInstanceController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(FORCE_TASK_SUCCESS_ERROR) @OperatorLog(auditType = AuditType.TASK_INSTANCE_FORCE_SUCCESS) - public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @Schema(name = "projectCode", required = true) @PathVariable long projectCode, - @PathVariable(value = "id") Integer id) { - return taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @Schema(name = "projectCode", required = true) @PathVariable long projectCode, + @PathVariable(value = "id") Integer id) { + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return Result.success(); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java index 3e3a87681b..ea767f0fa3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java @@ -167,8 +167,8 @@ public class TaskInstanceV2Controller extends BaseController { public TaskInstanceSuccessResponse forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "id") Integer id) { - Result result = taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); - return new TaskInstanceSuccessResponse(result); + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return new TaskInstanceSuccessResponse(Result.success()); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 86e5396dbe..bff0518685 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -72,9 +72,9 @@ public interface TaskInstanceService { * @param taskInstanceId task instance id * @return the result code and msg */ - Result forceTaskSuccess(User loginUser, - long projectCode, - Integer taskInstanceId); + void forceTaskSuccess(User loginUser, + long projectCode, + Integer taskInstanceId); /** * task savepoint diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index f06f8115a9..7469d8db13 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -23,6 +23,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.service.TaskGroupQueueService; import org.apache.dolphinscheduler.api.service.TaskInstanceService; @@ -33,14 +34,15 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.repository.DqExecuteResultDao; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils; import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; @@ -107,6 +109,9 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst @Autowired private TaskGroupQueueService taskGroupQueueService; + @Autowired + private ProcessInstanceDao workflowInstanceDao; + /** * query task list by project, process instance, task name, task start time, task end time, task status, keyword paging * @@ -216,58 +221,39 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst */ @Transactional @Override - public Result forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { - Result result = new Result(); - Project project = projectMapper.queryByCode(projectCode); + public void forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { // check user access for project - Map checkResult = - projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); - Status status = (Status) checkResult.get(Constants.STATUS); - if (status != Status.SUCCESS) { - putMsg(result, status); - return result; + projectService.checkProjectAndAuthThrowException(loginUser, projectCode, FORCED_SUCCESS); + + TaskInstance task = taskInstanceDao.queryOptionalById(taskInstanceId) + .orElseThrow(() -> new ServiceException(Status.TASK_INSTANCE_NOT_FOUND)); + + if (task.getProjectCode() != projectCode) { + throw new ServiceException("The task instance is not under the project: " + projectCode); } - // check whether the task instance can be found - TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); - if (task == null) { - log.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, - taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); - return result; - } - - TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(task.getTaskCode()); - if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - log.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, - task.getTaskCode()); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND, taskInstanceId); - return result; + ProcessInstance processInstance = workflowInstanceDao.queryOptionalById(task.getProcessInstanceId()) + .orElseThrow( + () -> new ServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, task.getProcessInstanceId())); + if (!processInstance.getState().isFinished()) { + throw new ServiceException("The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance"); } // check whether the task instance state type is failure or cancel if (!task.getState().isFailure() && !task.getState().isKill()) { - log.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", - task.getState().getDesc(), projectCode, taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState().toString()); - return result; + throw new ServiceException(Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState()); } // change the state of the task instance task.setState(TaskExecutionStatus.FORCED_SUCCESS); task.setEndTime(new Date()); int changedNum = taskInstanceMapper.updateById(task); - if (changedNum > 0) { - processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId); - log.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, - taskInstanceId); - putMsg(result, Status.SUCCESS); - } else { - log.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", - projectCode, taskInstanceId); - putMsg(result, Status.FORCE_TASK_SUCCESS_ERROR); + if (changedNum <= 0) { + throw new ServiceException(Status.FORCE_TASK_SUCCESS_ERROR); } - return result; + processService.forceProcessInstanceSuccessByTaskInstanceId(task); + log.info("Force success task instance:{} success", taskInstanceId); } @Override diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java index d2da5bc638..eae064bb24 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import java.text.MessageFormat; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; @@ -30,6 +32,12 @@ public class AssertionsHelper extends Assertions { Assertions.assertEquals(status.getCode(), exception.getCode()); } + public static void assertThrowsServiceException(String message, Executable executable) { + ServiceException exception = Assertions.assertThrows(ServiceException.class, executable); + Assertions.assertEquals(MessageFormat.format(Status.INTERNAL_SERVER_ERROR_ARGS.getMsg(), message), + exception.getMessage()); + } + public static void assertDoesNotThrow(Executable executable) { Assertions.assertDoesNotThrow(executable); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 7ebe5bf757..b58944537b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -82,9 +82,7 @@ public class TaskInstanceControllerTest extends AbstractControllerTest { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("taskInstanceId", "104"); - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); - when(taskInstanceService.forceTaskSuccess(any(User.class), anyLong(), anyInt())).thenReturn(mockResult); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(User.class), anyLong(), anyInt()); MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/task-instance/force-success", "cxc_1113") .header(SESSION_ID, sessionId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java index cc70f200e1..8e76ec1694 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.controller.AbstractControllerTest; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceQueryRequest; +import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceSuccessResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -85,12 +86,9 @@ public class TaskInstanceV2ControllerTest extends AbstractControllerTest { @Test public void testForceTaskSuccess() { - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt()); - when(taskInstanceService.forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt())).thenReturn(mockResult); - - Result taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); + TaskInstanceSuccessResponse taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); Assertions.assertEquals(Integer.valueOf(Status.SUCCESS.getCode()), taskResult.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java index aca0d80a6f..dd7acb16d5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.FORCED_SUCCESS; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_INSTANCE; import static org.mockito.ArgumentMatchers.any; @@ -25,7 +26,6 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; @@ -35,15 +35,16 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -65,7 +66,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.springframework.boot.test.context.SpringBootTest; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -74,7 +74,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -@SpringBootTest(classes = ApiApplicationServer.class) public class TaskInstanceServiceTest { @InjectMocks @@ -100,6 +99,8 @@ public class TaskInstanceServiceTest { @Mock TaskInstanceDao taskInstanceDao; + @Mock + ProcessInstanceDao workflowInstanceDao; @Test public void queryTaskListPaging() { @@ -324,6 +325,7 @@ public class TaskInstanceServiceTest { private TaskInstance getTaskInstance() { TaskInstance taskInstance = new TaskInstance(); taskInstance.setId(1); + taskInstance.setProjectCode(1L); taskInstance.setName("test_task_instance"); taskInstance.setStartTime(new Date()); taskInstance.setEndTime(new Date()); @@ -343,64 +345,69 @@ public class TaskInstanceServiceTest { } @Test - public void testForceTaskSuccess() { + public void testForceTaskSuccess_withNoPermission() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + .checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + assertThrowsServiceException(Status.USER_NO_OPERATION_PROJECT_PERM, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withTaskInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.empty()); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())).thenReturn(Optional.empty()); + + assertThrowsServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFinished() { User user = getAdminUser(); long projectCode = 1L; - Project project = getProject(projectCode); - int taskId = 1; TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, projectCode, FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); - Map mockSuccess = new HashMap<>(5); - putMsg(mockSuccess, Status.SUCCESS); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); + assertThrowsServiceException( + "The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance", + () -> taskInstanceService.forceTaskSuccess(user, projectCode, task.getId())); + } - // user auth failed - Map mockFailure = new HashMap<>(5); - putMsg(mockFailure, Status.USER_NO_OPERATION_PROJECT_PERM, user.getUserName(), projectCode); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockFailure); - Result authFailRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertNotSame(Status.SUCCESS.getCode(), authFailRes.getCode()); - - // test task not found - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockSuccess); - when(taskInstanceMapper.selectById(Mockito.anyInt())).thenReturn(null); - TaskDefinition taskDefinition = new TaskDefinition(); - taskDefinition.setProjectCode(projectCode); - when(taskDefinitionMapper.queryByCode(task.getTaskCode())).thenReturn(taskDefinition); - Result taskNotFoundRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), taskNotFoundRes.getCode().intValue()); - - // test task instance state error - task.setState(TaskExecutionStatus.SUCCESS); - when(taskInstanceMapper.selectById(1)).thenReturn(task); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result taskStateErrorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_STATE_OPERATION_ERROR.getCode(), - taskStateErrorRes.getCode().intValue()); - - // test error - task.setState(TaskExecutionStatus.FAILURE); - when(taskInstanceMapper.updateById(task)).thenReturn(0); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result errorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.FORCE_TASK_SUCCESS_ERROR.getCode(), errorRes.getCode().intValue()); - - // test success - task.setState(TaskExecutionStatus.FAILURE); - task.setEndTime(null); - when(taskInstanceMapper.updateById(task)).thenReturn(1); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result successRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.SUCCESS.getCode(), successRes.getCode().intValue()); - Assertions.assertNotNull(task.getEndTime()); + @Test + public void testForceTaskSuccess_withTaskInstanceNotFinished() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.FAILURE); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); + assertThrowsServiceException( + Status.TASK_INSTANCE_STATE_OPERATION_ERROR, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); } @Test diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 101350e90d..8787aabc8b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -189,7 +189,7 @@ public interface ProcessService { public String findConfigYamlByName(String clusterName); - void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId); + void forceProcessInstanceSuccessByTaskInstanceId(TaskInstance taskInstance); void saveCommandTrigger(Integer commandId, Integer processInstanceId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 028ab7651f..3c207ae982 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -276,6 +276,7 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private TriggerRelationService triggerRelationService; + /** * todo: split this method * handle Command (construct ProcessInstance from Command) , wrapped in transaction @@ -621,13 +622,13 @@ public class ProcessServiceImpl implements ProcessService { /** * Get workflow runtime tenant - * + *

* the workflow provides a tenant and uses the provided tenant; * when no tenant is provided or the provided tenant is the default tenant, \ * the user's tenant created by the workflow is used * * @param tenantCode tenantCode - * @param userId userId + * @param userId userId * @return tenant code */ @Override @@ -2114,11 +2115,7 @@ public class ProcessServiceImpl implements ProcessService { } @Override - public void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId) { - TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); - if (task == null) { - return; - } + public void forceProcessInstanceSuccessByTaskInstanceId(TaskInstance task) { ProcessInstance processInstance = findProcessInstanceDetailById(task.getProcessInstanceId()).orElse(null); if (processInstance != null && (processInstance.getState().isFailure() || processInstance.getState().isStop())) { @@ -2139,7 +2136,7 @@ public class ProcessServiceImpl implements ProcessService { List failTaskList = validTaskList.stream() .filter(instance -> instance.getState().isFailure() || instance.getState().isKill()) .map(TaskInstance::getId).collect(Collectors.toList()); - if (failTaskList.size() == 1 && failTaskList.contains(taskInstanceId)) { + if (failTaskList.size() == 1 && failTaskList.contains(task.getId())) { processInstance.setStateWithDesc(WorkflowExecutionStatus.SUCCESS, "success by task force success"); processInstanceDao.updateById(processInstance); } From ead54534a2061cc2daaf8ee24024a011a8b8ce4f Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 17:27:36 +0800 Subject: [PATCH 13/41] Fix QUARTZ table order is not correct in initialization schema (#15857) --- .../resources/sql/dolphinscheduler_mysql.sql | 130 +++++++++--------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index e2138b67a2..b30bc13887 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -15,7 +15,70 @@ * limitations under the License. */ -SET FOREIGN_KEY_CHECKS=0; +-- ---------------------------- +-- Table structure for QRTZ_JOB_DETAILS +-- ---------------------------- +DROP TABLE IF EXISTS `QRTZ_JOB_DETAILS`; +CREATE TABLE `QRTZ_JOB_DETAILS` ( + `SCHED_NAME` varchar(120) NOT NULL, + `JOB_NAME` varchar(200) NOT NULL, + `JOB_GROUP` varchar(200) NOT NULL, + `DESCRIPTION` varchar(250) DEFAULT NULL, + `JOB_CLASS_NAME` varchar(250) NOT NULL, + `IS_DURABLE` varchar(1) NOT NULL, + `IS_NONCONCURRENT` varchar(1) NOT NULL, + `IS_UPDATE_DATA` varchar(1) NOT NULL, + `REQUESTS_RECOVERY` varchar(1) NOT NULL, + `JOB_DATA` blob, + PRIMARY KEY (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_J_REQ_RECOVERY` (`SCHED_NAME`,`REQUESTS_RECOVERY`), + KEY `IDX_QRTZ_J_GRP` (`SCHED_NAME`,`JOB_GROUP`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; + +-- ---------------------------- +-- Records of QRTZ_JOB_DETAILS +-- ---------------------------- + +-- ---------------------------- +-- Table structure for QRTZ_TRIGGERS +-- ---------------------------- +DROP TABLE IF EXISTS `QRTZ_TRIGGERS`; +CREATE TABLE `QRTZ_TRIGGERS` ( + `SCHED_NAME` varchar(120) NOT NULL, + `TRIGGER_NAME` varchar(200) NOT NULL, + `TRIGGER_GROUP` varchar(200) NOT NULL, + `JOB_NAME` varchar(200) NOT NULL, + `JOB_GROUP` varchar(200) NOT NULL, + `DESCRIPTION` varchar(250) DEFAULT NULL, + `NEXT_FIRE_TIME` bigint(13) DEFAULT NULL, + `PREV_FIRE_TIME` bigint(13) DEFAULT NULL, + `PRIORITY` int(11) DEFAULT NULL, + `TRIGGER_STATE` varchar(16) NOT NULL, + `TRIGGER_TYPE` varchar(8) NOT NULL, + `START_TIME` bigint(13) NOT NULL, + `END_TIME` bigint(13) DEFAULT NULL, + `CALENDAR_NAME` varchar(200) DEFAULT NULL, + `MISFIRE_INSTR` smallint(2) DEFAULT NULL, + `JOB_DATA` blob, + PRIMARY KEY (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`), + KEY `IDX_QRTZ_T_J` (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_T_JG` (`SCHED_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_T_C` (`SCHED_NAME`,`CALENDAR_NAME`), + KEY `IDX_QRTZ_T_G` (`SCHED_NAME`,`TRIGGER_GROUP`), + KEY `IDX_QRTZ_T_STATE` (`SCHED_NAME`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_N_STATE` (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_N_G_STATE` (`SCHED_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_NEXT_FIRE_TIME` (`SCHED_NAME`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_ST` (`SCHED_NAME`,`TRIGGER_STATE`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_ST_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_NFT_ST_MISFIRE_GRP` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + CONSTRAINT `QRTZ_TRIGGERS_ibfk_1` FOREIGN KEY (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) REFERENCES `QRTZ_JOB_DETAILS` (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; + +-- ---------------------------- +-- Records of QRTZ_TRIGGERS +-- ---------------------------- -- ---------------------------- -- Table structure for QRTZ_BLOB_TRIGGERS @@ -99,30 +162,6 @@ CREATE TABLE `QRTZ_FIRED_TRIGGERS` ( -- Records of QRTZ_FIRED_TRIGGERS -- ---------------------------- --- ---------------------------- --- Table structure for QRTZ_JOB_DETAILS --- ---------------------------- -DROP TABLE IF EXISTS `QRTZ_JOB_DETAILS`; -CREATE TABLE `QRTZ_JOB_DETAILS` ( - `SCHED_NAME` varchar(120) NOT NULL, - `JOB_NAME` varchar(200) NOT NULL, - `JOB_GROUP` varchar(200) NOT NULL, - `DESCRIPTION` varchar(250) DEFAULT NULL, - `JOB_CLASS_NAME` varchar(250) NOT NULL, - `IS_DURABLE` varchar(1) NOT NULL, - `IS_NONCONCURRENT` varchar(1) NOT NULL, - `IS_UPDATE_DATA` varchar(1) NOT NULL, - `REQUESTS_RECOVERY` varchar(1) NOT NULL, - `JOB_DATA` blob, - PRIMARY KEY (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_J_REQ_RECOVERY` (`SCHED_NAME`,`REQUESTS_RECOVERY`), - KEY `IDX_QRTZ_J_GRP` (`SCHED_NAME`,`JOB_GROUP`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - --- ---------------------------- --- Records of QRTZ_JOB_DETAILS --- ---------------------------- - -- ---------------------------- -- Table structure for QRTZ_LOCKS -- ---------------------------- @@ -213,47 +252,6 @@ CREATE TABLE `QRTZ_SIMPROP_TRIGGERS` ( -- Records of QRTZ_SIMPROP_TRIGGERS -- ---------------------------- --- ---------------------------- --- Table structure for QRTZ_TRIGGERS --- ---------------------------- -DROP TABLE IF EXISTS `QRTZ_TRIGGERS`; -CREATE TABLE `QRTZ_TRIGGERS` ( - `SCHED_NAME` varchar(120) NOT NULL, - `TRIGGER_NAME` varchar(200) NOT NULL, - `TRIGGER_GROUP` varchar(200) NOT NULL, - `JOB_NAME` varchar(200) NOT NULL, - `JOB_GROUP` varchar(200) NOT NULL, - `DESCRIPTION` varchar(250) DEFAULT NULL, - `NEXT_FIRE_TIME` bigint(13) DEFAULT NULL, - `PREV_FIRE_TIME` bigint(13) DEFAULT NULL, - `PRIORITY` int(11) DEFAULT NULL, - `TRIGGER_STATE` varchar(16) NOT NULL, - `TRIGGER_TYPE` varchar(8) NOT NULL, - `START_TIME` bigint(13) NOT NULL, - `END_TIME` bigint(13) DEFAULT NULL, - `CALENDAR_NAME` varchar(200) DEFAULT NULL, - `MISFIRE_INSTR` smallint(2) DEFAULT NULL, - `JOB_DATA` blob, - PRIMARY KEY (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`), - KEY `IDX_QRTZ_T_J` (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_T_JG` (`SCHED_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_T_C` (`SCHED_NAME`,`CALENDAR_NAME`), - KEY `IDX_QRTZ_T_G` (`SCHED_NAME`,`TRIGGER_GROUP`), - KEY `IDX_QRTZ_T_STATE` (`SCHED_NAME`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_N_STATE` (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_N_G_STATE` (`SCHED_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_NEXT_FIRE_TIME` (`SCHED_NAME`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_ST` (`SCHED_NAME`,`TRIGGER_STATE`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_ST_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_NFT_ST_MISFIRE_GRP` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - CONSTRAINT `QRTZ_TRIGGERS_ibfk_1` FOREIGN KEY (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) REFERENCES `QRTZ_JOB_DETAILS` (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - --- ---------------------------- --- Records of QRTZ_TRIGGERS --- ---------------------------- - -- ---------------------------- -- Table structure for t_ds_access_token -- ---------------------------- From 9437d276e76b18a2cf6c469fa886a027115ece03 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 22:49:11 +0800 Subject: [PATCH 14/41] Change ssh heartbeat type to IGNORE (#15858) --- .../dolphinscheduler/plugin/datasource/ssh/SSHUtils.java | 2 +- .../plugin/task/remoteshell/RemoteExecutor.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java index 42e1175e2e..7abb18cee8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java @@ -59,7 +59,7 @@ public class SSHUtils { throw new Exception("Failed to add public key identity", e); } } - session.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.RESERVED, Duration.ofSeconds(3)); + session.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.IGNORE, Duration.ofSeconds(3)); return session; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java index 334ebb6195..307023043a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java @@ -141,7 +141,6 @@ public class RemoteExecutor implements AutoCloseable { } } cleanData(taskId); - log.error("Remote shell task failed"); return exitCode; } @@ -232,8 +231,10 @@ public class RemoteExecutor implements AutoCloseable { channel.open(); channel.waitFor(EnumSet.of(ClientChannelEvent.CLOSED), 0); channel.close(); - if (channel.getExitStatus() != 0) { - throw new TaskException("Remote shell task error, error message: " + err.toString()); + Integer exitStatus = channel.getExitStatus(); + if (exitStatus == null || exitStatus != 0) { + throw new TaskException( + "Remote shell task error, exitStatus: " + exitStatus + " error message: " + err); } return out.toString(); } From 9a48aca83c7275c934a08eab3c0bf4203a80149d Mon Sep 17 00:00:00 2001 From: zuo <58384836+xxzuo@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:18:29 +0800 Subject: [PATCH 15/41] [Fix-15866][Doc] update the taobao mirror link (#15867) --- docs/docs/en/contribute/frontend-development.md | 2 +- docs/docs/en/faq.md | 4 ++-- docs/docs/zh/contribute/frontend-development.md | 2 +- docs/docs/zh/faq.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/en/contribute/frontend-development.md b/docs/docs/en/contribute/frontend-development.md index 0cc86811f8..50f8c91fb4 100644 --- a/docs/docs/en/contribute/frontend-development.md +++ b/docs/docs/en/contribute/frontend-development.md @@ -33,7 +33,7 @@ Use the command line mode `cd` enter the `dolphinscheduler-ui` project director > If `npm install` is very slow, you can set the taobao mirror ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend: diff --git a/docs/docs/en/faq.md b/docs/docs/en/faq.md index b0954b3468..7ac4afb76e 100644 --- a/docs/docs/en/faq.md +++ b/docs/docs/en/faq.md @@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory sudo rm -rf node_modules ``` -​ 2, install node-sass through npm.taobao.org +​ 2, install node-sass through npmmirror.com ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md) diff --git a/docs/docs/zh/contribute/frontend-development.md b/docs/docs/zh/contribute/frontend-development.md index 42eb2973dd..249b17d546 100644 --- a/docs/docs/zh/contribute/frontend-development.md +++ b/docs/docs/zh/contribute/frontend-development.md @@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2 > 如果 `npm install` 速度非常慢，你可以设置淘宝镜像 ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互： diff --git a/docs/docs/zh/faq.md b/docs/docs/zh/faq.md index 13e0dc8b84..a4f32d08a8 100644 --- a/docs/docs/zh/faq.md +++ b/docs/docs/zh/faq.md @@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录 sudo rm -rf node_modules ``` -​ 2，通过 npm.taobao.org 下载 node-sass +​ 2，通过 npmmirror.com 下载 node-sass ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md) From 3fe9fd45b12e2e6d190729e64b2c4663b0752ada Mon Sep 17 00:00:00 2001 From: XinXing <49302071+xinxingi@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:36:18 +0800 Subject: [PATCH 16/41] [Fix-15706] Seatunnel improvement (#15852) * fix_seatunnel_15706 * CodeFormat * Change to use JSONUtils * Constants moved to constant code * Delete empty lines * Delete empty lines --- .../common/utils/JSONUtils.java | 6 +- .../plugin/task/seatunnel/Constants.java | 2 + .../plugin/task/seatunnel/SeatunnelTask.java | 9 +- .../task/seatunnel/SeatunnelTaskTest.java | 93 +++++++++++++++++++ 4 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 6750b364f9..bfc3af2c58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -196,7 +196,10 @@ public final class JSONUtils { * @return true if valid */ public static boolean checkJsonValid(String json) { + return checkJsonValid(json, true); + } + public static boolean checkJsonValid(String json, Boolean logFlag) { if (Strings.isNullOrEmpty(json)) { return false; } @@ -205,7 +208,8 @@ public final class JSONUtils { objectMapper.readTree(json); return true; } catch (IOException e) { - log.error("check json object valid exception!", e); + if (logFlag) + log.error("check json object valid exception!", e); } return false; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java index fb1c52ee18..5f0f22b0a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java @@ -29,5 +29,7 @@ public class Constants { public static final String STARTUP_SCRIPT_SPARK = "spark"; public static final String STARTUP_SCRIPT_FLINK = "flink"; public static final String STARTUP_SCRIPT_SEATUNNEL = "seatunnel"; + public static final String JSON_SUFFIX = "json"; + public static final String CONF_SUFFIX = "conf"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java index 547e0158ef..2aadab8039 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java @@ -184,8 +184,13 @@ public class SeatunnelTask extends AbstractRemoteTask { } private String buildConfigFilePath() { - return String.format("%s/seatunnel_%s.conf", taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId()); + return String.format("%s/seatunnel_%s.%s", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId(), formatDetector()); + } + + private String formatDetector() { + return JSONUtils.checkJsonValid(seatunnelParameters.getRawScript(), false) ? Constants.JSON_SUFFIX + : Constants.CONF_SUFFIX; } private void createConfigFileIfNotExists(String script, String scriptFile) throws IOException { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java new file mode 100644 index 0000000000..8f4c2a815a --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.task.seatunnel; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +public class SeatunnelTaskTest { + private static final String EXECUTE_PATH = "/home"; + private static final String TASK_APPID = "9527"; + + @Test + public void formatDetector() throws Exception{ + SeatunnelParameters seatunnelParameters = new SeatunnelParameters(); + seatunnelParameters.setRawScript(RAW_SCRIPT); + + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setExecutePath(EXECUTE_PATH); + taskExecutionContext.setTaskAppId(TASK_APPID); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(seatunnelParameters)); + + SeatunnelTask seatunnelTask = new SeatunnelTask(taskExecutionContext); + seatunnelTask.setSeatunnelParameters(seatunnelParameters); + Assertions.assertEquals("/home/seatunnel_9527.conf", seatunnelTask.buildCustomConfigCommand()); + + seatunnelParameters.setRawScript(RAW_SCRIPT_2); + seatunnelTask.setSeatunnelParameters(seatunnelParameters); + Assertions.assertEquals("/home/seatunnel_9527.json", seatunnelTask.buildCustomConfigCommand()); + } + private static final String RAW_SCRIPT = "env {\n" + + " execution.parallelism = 2\n" + + " job.mode = \"BATCH\"\n" + + " checkpoint.interval = 10000\n" + + "}\n" + + "\n" + + "source {\n" + + " FakeSource {\n" + + " parallelism = 2\n" + + " result_table_name = \"fake\"\n" + + " row.num = 16\n" + + " schema = {\n" + + " fields {\n" + + " name = \"string\"\n" + + " age = \"int\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "\n" + + "sink {\n" + + " Console {\n" + + " }\n" + + "}"; + private static final String RAW_SCRIPT_2 = "{\n" + + " \"env\": {\n" + + " \"execution.parallelism\": 2,\n" + + " \"job.mode\": \"BATCH\",\n" + + " \"checkpoint.interval\": 10000\n" + + " },\n" + + " \"source\": {\n" + + " \"FakeSource\": {\n" + + " \"parallelism\": 2,\n" + + " \"result_table_name\": \"fake\",\n" + + " \"row.num\": 16,\n" + + " \"schema\": {\n" + + " \"fields\": {\n" + + " \"name\": \"string\",\n" + + " \"age\": \"int\"\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sink\": {\n" + + " \"Console\": {}\n" + + " }\n" + + "}"; +} \ No newline at end of file From a8bc23748d7e4371e962c095d40e06ce88fdf497 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 11:31:11 +0800 Subject: [PATCH 17/41] Fix write audit log error will cause the request failed (#15868) --- .../api/audit/OperatorLogAspect.java | 16 ++++++++++-- .../api/audit/OperatorUtils.java | 1 - .../api/audit/operator/AuditOperator.java | 13 +++++++--- .../api/audit/operator/BaseAuditOperator.java | 26 +++++++++---------- .../api/service/AuditService.java | 10 ------- .../api/service/impl/AuditServiceImpl.java | 8 ------ 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java index 4e15112980..da8a7bd3e6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.api.audit; import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.lang.reflect.Method; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -54,8 +56,18 @@ public class OperatorLogAspect { log.warn("Operation is null of method: {}", method.getName()); return point.proceed(); } + long beginTime = System.currentTimeMillis(); - AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); - return operator.recordAudit(point, operation.description(), operatorLog.auditType()); + Map paramsMap = OperatorUtils.getParamsMap(point, signature); + Result result = (Result) point.proceed(); + try { + AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); + long latency = System.currentTimeMillis() - beginTime; + operator.recordAudit(paramsMap, result, latency, operation, operatorLog); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } + + return result; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index a0233c7e93..10ccf8d648 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -66,7 +66,6 @@ public class OperatorUtils { auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); auditLog.setCreateTime(new Date()); - auditLogList.add(auditLog); return auditLogList; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java index 6e1c016313..7d76d5c4bd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java @@ -17,11 +17,18 @@ package org.apache.dolphinscheduler.api.audit.operator; -import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.utils.Result; -import org.aspectj.lang.ProceedingJoinPoint; +import java.util.Map; + +import io.swagger.v3.oas.annotations.Operation; public interface AuditOperator { - Object recordAudit(ProceedingJoinPoint point, String describe, AuditType auditType) throws Throwable; + void recordAudit(Map paramsMap, + Result result, + long latency, + Operation operation, + OperatorLog operatorLog) throws Throwable; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java index 5c896a058a..270a5e4df4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.audit.operator; +import org.apache.dolphinscheduler.api.audit.OperatorLog; import org.apache.dolphinscheduler.api.audit.OperatorUtils; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.service.AuditService; @@ -31,12 +32,11 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.reflect.MethodSignature; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.base.Strings; +import io.swagger.v3.oas.annotations.Operation; @Service @Slf4j @@ -46,26 +46,27 @@ public abstract class BaseAuditOperator implements AuditOperator { private AuditService auditService; @Override - public Object recordAudit(ProceedingJoinPoint point, String describe, AuditType auditType) throws Throwable { - long beginTime = System.currentTimeMillis(); + public void recordAudit(Map paramsMap, + Result result, + long latency, + Operation operation, + OperatorLog operatorLog) { - MethodSignature signature = (MethodSignature) point.getSignature(); - Map paramsMap = OperatorUtils.getParamsMap(point, signature); + AuditType auditType = operatorLog.auditType(); User user = OperatorUtils.getUser(paramsMap); if (user == null) { log.error("user is null"); - return point.proceed(); + return; } - List auditLogList = OperatorUtils.buildAuditLogList(describe, auditType, user); + List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); setRequestParam(auditType, auditLogList, paramsMap); - Result result = (Result) point.proceed(); if (OperatorUtils.resultFail(result)) { log.error("request fail, code {}", result.getCode()); - return result; + return; } setObjectIdentityFromReturnObject(auditType, result, auditLogList); @@ -73,10 +74,9 @@ public abstract class BaseAuditOperator implements AuditOperator { modifyAuditOperationType(auditType, paramsMap, auditLogList); modifyAuditObjectType(auditType, paramsMap, auditLogList); - long latency = System.currentTimeMillis() - beginTime; - auditService.addAudit(auditLogList, latency); + auditLogList.forEach(auditLog -> auditLog.setLatency(latency)); + auditLogList.forEach(auditLog -> auditService.addAudit(auditLog)); - return result; } protected void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java index b4296452f6..8fb99eaf73 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.api.dto.AuditDto; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import java.util.List; - /** * audit information service */ @@ -35,14 +33,6 @@ public interface AuditService { */ void addAudit(AuditLog auditLog); - /** - * add audit by list - * - * @param auditLogList auditLog list - * @param latency api latency milliseconds - */ - void addAudit(List auditLogList, long latency); - /** * query audit log list * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index 2ec547bcb6..f8f763b4f4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -57,14 +57,6 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { auditLogMapper.insert(auditLog); } - @Override - public void addAudit(List auditLogList, long latency) { - auditLogList.forEach(auditLog -> { - auditLog.setLatency(latency); - addAudit(auditLog); - }); - } - /** * query audit log paging * From 163f8f01f0b3726374c81d9fa87da6571b3c460c Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 15:43:28 +0800 Subject: [PATCH 18/41] Fix jdbc registry cannot work (#15861) --- .github/workflows/backend.yml | 12 +- .../Dockerfile | 6 +- .../mysql_with_mysql_registry/deploy.sh | 44 +++++ .../docker-compose-base.yaml | 34 ++++ .../docker-compose-cluster.yaml | 0 .../dolphinscheduler_env.sh | 58 +++++++ .../mysql_with_mysql_registry/install_env.sh | 58 +++++++ .../running_test.sh | 0 .../start-job.sh | 8 +- .../mysql_with_zookeeper_registry/Dockerfile | 48 ++++++ .../deploy.sh | 0 .../docker-compose-base.yaml | 0 .../docker-compose-cluster.yaml | 29 ++++ .../dolphinscheduler_env.sh | 0 .../install_env.sh | 0 .../running_test.sh | 108 +++++++++++++ .../start-job.sh | 33 ++++ .../Dockerfile | 39 +++++ .../deploy.sh | 41 +++++ .../docker-compose-base.yaml | 35 ++++ .../docker-compose-cluster.yaml | 0 .../dolphinscheduler_env.sh | 58 +++++++ .../install_env.sh | 58 +++++++ .../running_test.sh | 0 .../start-job.sh | 33 ++++ .../Dockerfile | 6 +- .../deploy.sh | 0 .../docker-compose-base.yaml | 0 .../docker-compose-cluster.yaml | 29 ++++ .../dolphinscheduler_env.sh | 0 .../install_env.sh | 0 .../running_test.sh | 109 +++++++++++++ .../start-job.sh | 8 +- .../dolphinscheduler/alert/AlertServer.java | 19 ++- .../api/ApiApplicationServer.java | 6 +- .../dolphinscheduler/dao/PluginDao.java | 4 +- .../{ => repository/impl}/AlertDaoTest.java | 4 +- .../server/master/MasterServer.java | 6 +- .../dolphinscheduler-registry-jdbc/README.md | 10 +- .../dolphinscheduler-registry-jdbc/pom.xml | 10 ++ ...ava => JdbcRegistryAutoConfiguration.java} | 30 +++- .../main/resources/META-INF/spring.factories | 19 +++ .../main/resources/mysql_registry_init.sql | 1 - .../src/main/bin/initialize-jdbc-registry.sh | 31 ++++ .../tools/command/CommandApplication.java | 152 ++++++++++++++++++ .../src/main/resources/application.yaml | 2 + .../server/worker/WorkerServer.java | 6 +- 47 files changed, 1111 insertions(+), 43 deletions(-) rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/Dockerfile (85%) create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/docker-compose-cluster.yaml (100%) create mode 100755 .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/running_test.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/start-job.sh (74%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/deploy.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/docker-compose-base.yaml (100%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/dolphinscheduler_env.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/install_env.sh (100%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml rename .github/workflows/cluster-test/{postgresql => postgresql_with_postgresql_registry}/docker-compose-cluster.yaml (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_postgresql_registry}/running_test.sh (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/Dockerfile (77%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/deploy.sh (100%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/docker-compose-base.yaml (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/dolphinscheduler_env.sh (100%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/install_env.sh (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/start-job.sh (73%) rename dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/{ => repository/impl}/AlertDaoTest.java (95%) rename dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/{JdbcRegistryConfiguration.java => JdbcRegistryAutoConfiguration.java} (65%) create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh create mode 100644 dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ea09a17fd2..671223d286 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -106,10 +106,14 @@ jobs: strategy: matrix: case: - - name: cluster-test-mysql - script: .github/workflows/cluster-test/mysql/start-job.sh - - name: cluster-test-postgresql - script: .github/workflows/cluster-test/postgresql/start-job.sh + - name: cluster-test-mysql-with-zookeeper-registry + script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh + - name: cluster-test-mysql-with-mysql-registry + script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh + - name: cluster-test-postgresql-zookeeper-registry + script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh + - name: cluster-test-postgresql-with-postgresql-registry + script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/cluster-test/mysql/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile similarity index 85% rename from .github/workflows/cluster-test/mysql/Dockerfile rename to .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile index c7d6abe889..12c7db3c18 100644 --- a/.github/workflows/cluster-test/mysql/Dockerfile +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile @@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Download mysql jar ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" @@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER #Deploy -COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh new file mode 100644 index 0000000000..72b2a630fa --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Create database +mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..d59e3c868c --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + mysql: + container_name: mysql + image: mysql:5.7.36 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: 123456 + ports: + - "3306:3306" + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + timeout: 60s + retries: 120 diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh new file mode 100755 index 0000000000..58937e740c --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export SPRING_DATASOURCE_USERNAME=root +export SPRING_DATASOURCE_PASSWORD=123456 + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export REGISTRY_HIKARI_CONFIG_USERNAME=root +export REGISTRY_HIKARI_CONFIG_PASSWORD=123456 + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql/running_test.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/running_test.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/mysql/start-job.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh similarity index 74% rename from .github/workflows/cluster-test/mysql/start-job.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh index ee67c5179b..0ce48c64ae 100644 --- a/.github/workflows/cluster-test/mysql/start-job.sh +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d #Build ds mysql cluster image -docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile . +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile . #Start ds mysql cluster container -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/mysql/running_test.sh +/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile new file mode 100644 index 0000000000..574c059442 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Download mysql jar +ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER + +#Deploy +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql/deploy.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/deploy.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/mysql/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-base.yaml rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..7343c8eee7 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_mysql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/mysql/install_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/install_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..7582c3ccc5 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh new file mode 100644 index 0000000000..db8d23147e --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d + +#Build ds mysql cluster image +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile . + +#Start ds mysql cluster container +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile new file mode 100644 index 0000000000..bb2d9a5383 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Deploy +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh new file mode 100644 index 0000000000..37bf3433c0 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..1793d94f39 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + postgres: + container_name: postgres + image: postgres:14.1 + restart: always + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dolphinscheduler + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh new file mode 100644 index 0000000000..e7fd1b7204 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-postgresql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export SPRING_DATASOURCE_USERNAME=postgres +export SPRING_DATASOURCE_PASSWORD=postgres + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=jdbc +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export REGISTRY_HIKARI_CONFIG_USERNAME=postgres +export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/running_test.sh rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh new file mode 100644 index 0000000000..e2b6b630e8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d + +#Build ds postgresql cluster image +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile . + +#Start ds postgresql cluster container +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile similarity index 77% rename from .github/workflows/cluster-test/postgresql/Dockerfile rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile index 38234ee7b3..077b5c97b8 100644 --- a/.github/workflows/cluster-test/postgresql/Dockerfile +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile @@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Deploy -COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql/deploy.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/deploy.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-base.yaml rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..9ab79ea44d --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_postgresql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/install_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..0bc861c389 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh similarity index 73% rename from .github/workflows/cluster-test/postgresql/start-job.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh index ba0878e3ec..fe755c97f1 100644 --- a/.github/workflows/cluster-test/postgresql/start-job.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d #Build ds postgresql cluster image -docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile . +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile . #Start ds postgresql cluster container -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh +/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index fd3d4b02e3..ff0088ea8d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -27,21 +27,24 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.event.EventListener; +import org.springframework.context.annotation.FilterType; -@SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") @Slf4j +@SpringBootApplication +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) public class AlertServer { @Autowired @@ -59,11 +62,11 @@ public class AlertServer { AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); Thread.currentThread().setName(Constants.THREAD_NAME_ALERT_SERVER); - new SpringApplicationBuilder(AlertServer.class).run(args); + SpringApplication.run(AlertServer.class, args); } - @EventListener - public void run(ApplicationReadyEvent readyEvent) { + @PostConstruct + public void run() { log.info("Alert server is staring ..."); alertPluginManager.start(); alertRegistryClient.start(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index c7e6d9778f..20a6412076 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; @@ -38,11 +39,14 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.servlet.ServletComponentScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.context.event.EventListener; @ServletComponentScan @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @Slf4j public class ApiApplicationServer { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java index 71e3be70c4..24cb022881 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java @@ -29,10 +29,10 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Repository; @Slf4j -@Component +@Repository public class PluginDao { @Autowired diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java similarity index 95% rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java rename to dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index f2cb503d9b..c0a841c1a0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao; +package org.apache.dolphinscheduler.dao.repository.impl; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.ProfileType; +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.entity.Alert; import java.util.List; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 752479e600..fd262a8c6e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.scheduler.api.SchedulerApi; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; @@ -46,10 +47,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @EnableTransactionManagement @EnableCaching @Slf4j diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md index 3b1a2cb24f..554c375218 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md @@ -1,6 +1,7 @@ # Introduction -This module is the jdbc registry plugin module, this plugin will use jdbc as the registry center. Will use the database configuration same as DolphinScheduler in api'yaml default. +This module is the jdbc registry plugin module, this plugin will use jdbc as the registry center. Will use the database +configuration same as DolphinScheduler in api'yaml default. # How to use @@ -22,8 +23,11 @@ registry: After do this two steps, you can start your DolphinScheduler cluster, your cluster will use mysql as registry center to store server metadata. -NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use mysql database, since this plugin will not bundle this driver in distribution. -You can get the detail about Initialize the Database. +NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use mysql database, since this plugin will not +bundle this driver in distribution. +You can get the detail +about Initialize the +Database. ## Optional configuration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml index 47b6449293..d4285edfbd 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml @@ -62,6 +62,16 @@ mybatis-plus + + com.baomidou + mybatis-plus-boot-starter + + + org.apache.logging.log4j + log4j-to-slf4j + + + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java similarity index 65% rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java index 7b37749ab7..09211f99fb 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java @@ -22,40 +22,56 @@ import org.apache.dolphinscheduler.plugin.registry.jdbc.mapper.JdbcRegistryLockM import org.apache.ibatis.session.SqlSessionFactory; +import lombok.extern.slf4j.Slf4j; + import org.mybatis.spring.SqlSessionTemplate; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration; import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import com.zaxxer.hikari.HikariDataSource; -@Configuration +@Slf4j +@Configuration(proxyBeanMethods = false) +@MapperScan("org.apache.dolphinscheduler.plugin.registry.jdbc.mapper") @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") -public class JdbcRegistryConfiguration { +@AutoConfigureAfter(MybatisPlusAutoConfiguration.class) +public class JdbcRegistryAutoConfiguration { + + public JdbcRegistryAutoConfiguration() { + log.info("Load JdbcRegistryAutoConfiguration"); + } @Bean - @ConditionalOnProperty(prefix = "registry.hikari-config", name = "jdbc-url") - public SqlSessionFactory jdbcRegistrySqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties) throws Exception { + @ConditionalOnMissingBean + public SqlSessionFactory sqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties) throws Exception { + log.info("Initialize jdbcRegistrySqlSessionFactory"); MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); sqlSessionFactoryBean.setDataSource(new HikariDataSource(jdbcRegistryProperties.getHikariConfig())); return sqlSessionFactoryBean.getObject(); } @Bean - public SqlSessionTemplate jdbcRegistrySqlSessionTemplate(SqlSessionFactory jdbcRegistrySqlSessionFactory) { - jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryDataMapper.class); - jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryLockMapper.class); + @ConditionalOnMissingBean + public SqlSessionTemplate sqlSessionTemplate(SqlSessionFactory jdbcRegistrySqlSessionFactory) { + log.info("Initialize jdbcRegistrySqlSessionTemplate"); return new SqlSessionTemplate(jdbcRegistrySqlSessionFactory); } @Bean public JdbcRegistryDataMapper jdbcRegistryDataMapper(SqlSessionTemplate jdbcRegistrySqlSessionTemplate) { + jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryDataMapper.class); return jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryDataMapper.class); } @Bean public JdbcRegistryLockMapper jdbcRegistryLockMapper(SqlSessionTemplate jdbcRegistrySqlSessionTemplate) { + jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryLockMapper.class); return jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryLockMapper.class); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..aabe7e3252 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql index 30af3066ff..6df206b391 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql @@ -15,7 +15,6 @@ * limitations under the License. */ -SET FOREIGN_KEY_CHECKS = 0; DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`; CREATE TABLE `t_ds_jdbc_registry_data` diff --git a/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh new file mode 100644 index 0000000000..895d62a0bb --- /dev/null +++ b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=${DATABASE} \ + org.apache.dolphinscheduler.tools.command.CommandApplication diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java new file mode 100644 index 0000000000..1e419f4d19 --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.command; + +import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Component; + +import com.baomidou.mybatisplus.annotation.DbType; + +// todo: use spring-shell to manage the command +@SpringBootApplication +@ImportAutoConfiguration(DaoConfiguration.class) +public class CommandApplication { + + public static void main(String[] args) { + SpringApplication.run(CommandApplication.class, args); + } + + @Component + @Slf4j + static class JdbcRegistrySchemaInitializeCommand implements CommandLineRunner { + + @Autowired + private DatabaseDialect databaseDialect; + + @Autowired + private DbType dbType; + + @Autowired + private DataSource dataSource; + + JdbcRegistrySchemaInitializeCommand() { + } + + @Override + public void run(String... args) throws Exception { + if (databaseDialect.tableExists("t_ds_jdbc_registry_data") + || databaseDialect.tableExists("t_ds_jdbc_registry_lock")) { + log.warn("t_ds_jdbc_registry_data/t_ds_jdbc_registry_lock already exists"); + return; + } + if (dbType == DbType.MYSQL) { + jdbcRegistrySchemaInitializeInMysql(); + } else if (dbType == DbType.POSTGRE_SQL) { + jdbcRegistrySchemaInitializeInPG(); + } else { + log.error("Unsupported database type: {}", dbType); + } + } + + private void jdbcRegistrySchemaInitializeInMysql() throws SQLException { + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE `t_ds_jdbc_registry_data`\n" + + "(\n" + + " `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',\n" + + " `data_key` varchar(256) NOT NULL COMMENT 'key, like zookeeper node path',\n" + + " `data_value` text NOT NULL COMMENT 'data, like zookeeper node value',\n" + + " `data_type` tinyint(4) NOT NULL COMMENT '1: ephemeral node, 2: persistent node',\n" + + + " `last_term` bigint NOT NULL COMMENT 'last term time',\n" + + " `last_update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n" + + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',\n" + + + " PRIMARY KEY (`id`),\n" + + " unique (`data_key`)\n" + + ") ENGINE = InnoDB\n" + + " DEFAULT CHARSET = utf8;"); + + statement.execute("CREATE TABLE `t_ds_jdbc_registry_lock`\n" + + "(\n" + + " `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',\n" + + " `lock_key` varchar(256) NOT NULL COMMENT 'lock path',\n" + + " `lock_owner` varchar(256) NOT NULL COMMENT 'the lock owner, ip_processId',\n" + + " `last_term` bigint NOT NULL COMMENT 'last term time',\n" + + " `last_update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n" + + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',\n" + + + " PRIMARY KEY (`id`),\n" + + " unique (`lock_key`)\n" + + ") ENGINE = InnoDB\n" + + " DEFAULT CHARSET = utf8;"); + } + } + + private void jdbcRegistrySchemaInitializeInPG() throws SQLException { + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("create table t_ds_jdbc_registry_data\n" + + "(\n" + + " id serial\n" + + " constraint t_ds_jdbc_registry_data_pk primary key,\n" + + " data_key varchar not null,\n" + + " data_value text not null,\n" + + " data_type int4 not null,\n" + + " last_term bigint not null,\n" + + " last_update_time timestamp default current_timestamp not null,\n" + + " create_time timestamp default current_timestamp not null\n" + + ");"); + statement.execute( + "create unique index t_ds_jdbc_registry_data_key_uindex on t_ds_jdbc_registry_data (data_key);"); + statement.execute("create table t_ds_jdbc_registry_lock\n" + + "(\n" + + " id serial\n" + + " constraint t_ds_jdbc_registry_lock_pk primary key,\n" + + " lock_key varchar not null,\n" + + " lock_owner varchar not null,\n" + + " last_term bigint not null,\n" + + " last_update_time timestamp default current_timestamp not null,\n" + + " create_time timestamp default current_timestamp not null\n" + + ");"); + statement.execute( + "create unique index t_ds_jdbc_registry_lock_key_uindex on t_ds_jdbc_registry_lock (lock_key);"); + } + } + + } +} diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml index 38752021dc..136a4c5fd4 100644 --- a/dolphinscheduler-tools/src/main/resources/application.yaml +++ b/dolphinscheduler-tools/src/main/resources/application.yaml @@ -63,6 +63,8 @@ spring: datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + username: root + password: root --- spring: diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 86e755fc8a..b0866f7fd8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; @@ -47,11 +48,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; @SpringBootApplication @EnableTransactionManagement -@ComponentScan(basePackages = "org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @Slf4j public class WorkerServer implements IStoppable { From 8fc204940f7be5cf356f63dd223bc599f857fe69 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 16:50:50 +0800 Subject: [PATCH 19/41] Add DSIP template (#15871) --- .github/ISSUE_TEMPLATE/dsip-request.yml | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/dsip-request.yml diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml new file mode 100644 index 0000000000..0908bba9e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dsip-request.yml @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Feature request +description: Suggest an idea for this project +title: "[DSIP-][Module Name] DSIP title" +labels: [ "DSIP", "Waiting for reply" ] +body: + - type: markdown + attributes: + value: | + For better global communication, Please write in English. + + If you feel the description in English is not clear, then you can append description in Chinese, thanks! + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first + to see whether the same DSIP was created already. + options: + - label: > + I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no + similar DSIP. + required: true + + - type: textarea + attributes: + label: Motivation + description: Why you want to do this change? + + - type: textarea + attributes: + label: Design Detail + description: Your design. + placeholder: > + It's better to provide a detailed design, such as the design of the interface, the design of the database, etc. + + - type: textarea + attributes: + label: Compatibility, Deprecation, and Migration Plan + description: > + If this feature is related to compatibility, deprecation, or migration, please describe it here. + + - type: textarea + attributes: + label: Test Plan + description: > + How to test this improvement. + + - type: checkboxes + attributes: + label: Code of Conduct + description: | + The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. + options: + - label: | + I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) + required: true + + - type: markdown + attributes: + value: "Thanks for completing our form!" From 325bfa821f9cd82ee15b681e8a5ae30d4976f8f1 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Thu, 18 Apr 2024 20:40:05 +0800 Subject: [PATCH 20/41] [TEST] increase cov of logger service (#15870) Co-authored-by: abzymeinsjtu Co-authored-by: Eric Gao --- .../api/service/impl/LoggerServiceImpl.java | 3 +- .../api/service/LoggerServiceTest.java | 62 ++++++++++++++++++- .../api/utils/ServiceTestUtil.java | 18 ++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index 0663b88374..c0ecb0e9b5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -237,7 +237,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService host, Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - byte[] logBytes = new byte[0]; + byte[] logBytes; ILogService iLogService = SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(), ILogService.class); @@ -251,6 +251,5 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService log.error("Download TaskInstance: {} Log Error", taskInstance.getName(), ex); throw new ServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR); } - } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 2c4de2ab7e..4861e1004e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -18,8 +18,10 @@ package org.apache.dolphinscheduler.api.service; import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; @@ -109,11 +111,25 @@ public class LoggerServiceTest { @Override public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { - return new TaskInstanceLogFileDownloadResponse(new byte[0]); + if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 1) { + return new TaskInstanceLogFileDownloadResponse(new byte[0]); + } else if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogFileDownloadResponse("log content".getBytes()); + } + + throw new ServiceException("download error"); } @Override public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() != null) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogPageQueryResponse("log content"); + } + } + return new TaskInstanceLogPageQueryResponse(); } @@ -177,6 +193,13 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); result = loggerService.queryLog(loginUser, 1, 1, 1); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + result = loggerService.queryLog(loginUser, 1, 0, 1); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + taskInstance.setLogPath(""); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, 1, 1, 1)); } @Test @@ -237,9 +260,15 @@ public class LoggerServiceTest { loginUser.setUserType(UserType.GENERAL_USER); TaskInstance taskInstance = new TaskInstance(); when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); + when(taskInstanceDao.queryById(10)).thenReturn(null); + + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); + TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(projectCode); taskDefinition.setCode(1L); + // SUCCESS taskInstance.setTaskCode(1L); taskInstance.setId(1); @@ -249,13 +278,27 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(10); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(1); + taskInstance.setId(10); + when(taskInstanceDao.queryById(10)).thenReturn(taskInstance); + String result = loggerService.queryLog(loginUser, projectCode, 10, 1, 1); + assertEquals("log content", result); + + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); } @Test public void testGetLogBytesInSpecifiedProject() { long projectCode = 1L; when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Project project = getProject(projectCode); User loginUser = new User(); loginUser.setId(-1); @@ -272,9 +315,24 @@ public class LoggerServiceTest { taskInstance.setHost("127.0.0.1:" + nettyServerPort); taskInstance.setLogPath("/temp/log"); doNothing().when(projectService).checkProjectAndAuthThrowException(loginUser, projectCode, DOWNLOAD_LOG); + + when(taskInstanceDao.queryById(1)).thenReturn(null); + assertThrowsServiceException( + Status.INTERNAL_SERVER_ERROR_ARGS, () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(2L); + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(1L); + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR, + () -> loggerService.getLogBytes(loginUser, projectCode, 100)); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java index 0dc71841b4..f33571d309 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.utils; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.User; + import java.nio.charset.StandardCharsets; import java.util.Random; @@ -27,4 +30,19 @@ public class ServiceTestUtil { new Random().nextBytes(bitArray); return new String(bitArray, StandardCharsets.UTF_8); } + + private static User getUser(Integer userId, String userName, UserType userType) { + User user = new User(); + user.setUserType(userType); + user.setId(userId); + user.setUserName(userName); + return user; + } + + public static User getAdminUser() { + return getUser(1, "admin", UserType.ADMIN_USER); + } + public static User getGeneralUser() { + return getUser(10, "user", UserType.GENERAL_USER); + } } From 32f92889746ccff0ee0379fd1b17ebe49a0591a7 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 19 Apr 2024 10:31:34 +0800 Subject: [PATCH 21/41] Fix dsip name (#15876) --- .github/ISSUE_TEMPLATE/dsip-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml index 0908bba9e7..f54421b066 100644 --- a/.github/ISSUE_TEMPLATE/dsip-request.yml +++ b/.github/ISSUE_TEMPLATE/dsip-request.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: Feature request +name: DSIP description: Suggest an idea for this project title: "[DSIP-][Module Name] DSIP title" labels: [ "DSIP", "Waiting for reply" ] From d306f1d04b3b4259093d139a28b2ad4d55d1dcf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Fri, 19 Apr 2024 15:20:31 +0800 Subject: [PATCH 22/41] Refactor record audit log logic (#15881) --- .../api/audit/OperatorLogAspect.java | 74 +++++++++++++++---- .../api/audit/OperatorUtils.java | 25 ++----- .../api/audit/operator/AuditOperator.java | 16 ++-- .../api/audit/operator/BaseAuditOperator.java | 43 +++++------ .../impl/TaskDefinitionServiceImpl.java | 6 +- .../common/constants/Constants.java | 4 + .../common/enums/AuditModelType.java | 2 +- 7 files changed, 100 insertions(+), 70 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java index da8a7bd3e6..aaf35d5b66 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java @@ -17,18 +17,26 @@ package org.apache.dolphinscheduler.api.audit; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; -import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.lang.reflect.Method; +import java.util.List; import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.AfterReturning; +import org.aspectj.lang.annotation.AfterThrowing; import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; import org.aspectj.lang.annotation.Pointcut; import org.aspectj.lang.reflect.MethodSignature; import org.springframework.stereotype.Component; @@ -40,34 +48,74 @@ import io.swagger.v3.oas.annotations.Operation; @Component public class OperatorLogAspect { + private static final ThreadLocal auditThreadLocal = new ThreadLocal<>(); + @Pointcut("@annotation(org.apache.dolphinscheduler.api.audit.OperatorLog)") public void logPointCut() { } - @Around("logPointCut()") - public Object around(ProceedingJoinPoint point) throws Throwable { + @Before("logPointCut()") + public void before(JoinPoint point) { MethodSignature signature = (MethodSignature) point.getSignature(); Method method = signature.getMethod(); - OperatorLog operatorLog = method.getAnnotation(OperatorLog.class); - Operation operation = method.getAnnotation(Operation.class); + if (operation == null) { log.warn("Operation is null of method: {}", method.getName()); - return point.proceed(); + return; } - long beginTime = System.currentTimeMillis(); Map paramsMap = OperatorUtils.getParamsMap(point, signature); - Result result = (Result) point.proceed(); + User user = OperatorUtils.getUser(paramsMap); + if (user == null) { + log.error("user is null"); + return; + } + + AuditType auditType = operatorLog.auditType(); + try { AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); - long latency = System.currentTimeMillis() - beginTime; - operator.recordAudit(paramsMap, result, latency, operation, operatorLog); + List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); + operator.setRequestParam(auditType, auditLogList, paramsMap); + AuditContext auditContext = + new AuditContext(auditLogList, paramsMap, operatorLog, System.currentTimeMillis(), operator); + auditThreadLocal.set(auditContext); } catch (Throwable throwable) { log.error("Record audit log error", throwable); } + } - return result; + @AfterReturning(value = "logPointCut()", returning = "returnValue") + public void afterReturning(Object returnValue) { + try { + AuditContext auditContext = auditThreadLocal.get(); + if (auditContext == null) { + return; + } + auditContext.getOperator().recordAudit(auditContext, returnValue); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } finally { + auditThreadLocal.remove(); + } + } + + @AfterThrowing("logPointCut()") + public void afterThrowing() { + auditThreadLocal.remove(); + } + + @Getter + @Setter + @AllArgsConstructor + public static class AuditContext { + + List auditLogList; + Map paramsMap; + OperatorLog operatorLog; + long beginTime; + AuditOperator operator; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index 10ccf8d648..8dc628b576 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.audit; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuditModelType; import org.apache.dolphinscheduler.common.enums.AuditOperationType; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -36,24 +37,12 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.JoinPoint; import org.aspectj.lang.reflect.MethodSignature; @Slf4j public class OperatorUtils { - protected void changeObjectForVersionRelated(AuditOperationType auditOperationType, Map paramsMap, - List auditLogList) { - switch (auditOperationType) { - case SWITCH_VERSION: - case DELETE_VERSION: - auditLogList.get(0).setModelName(paramsMap.get("version").toString()); - break; - default: - break; - } - } - public static boolean resultFail(Result result) { return result != null && result.isFailed(); } @@ -66,7 +55,7 @@ public class OperatorUtils { auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); auditLog.setCreateTime(new Date()); - + auditLogList.add(auditLog); return auditLogList; } @@ -80,7 +69,7 @@ public class OperatorUtils { return null; } - public static Map getParamsMap(ProceedingJoinPoint point, MethodSignature signature) { + public static Map getParamsMap(JoinPoint point, MethodSignature signature) { Object[] args = point.getArgs(); String[] strings = signature.getParameterNames(); @@ -95,7 +84,7 @@ public class OperatorUtils { public static AuditOperationType modifyReleaseOperationType(AuditType auditType, Map paramsMap) { switch (auditType.getAuditOperationType()) { case RELEASE: - ReleaseState releaseState = (ReleaseState) paramsMap.get("releaseState"); + ReleaseState releaseState = (ReleaseState) paramsMap.get(Constants.RELEASE_STATE); if (releaseState == null) { break; } @@ -109,7 +98,7 @@ public class OperatorUtils { } break; case EXECUTE: - ExecuteType executeType = (ExecuteType) paramsMap.get("executeType"); + ExecuteType executeType = (ExecuteType) paramsMap.get(Constants.EXECUTE_TYPE); if (executeType == null) { break; } @@ -184,7 +173,7 @@ public class OperatorUtils { } public static boolean isUdfResource(Map paramsMap) { - ResourceType resourceType = (ResourceType) paramsMap.get("type"); + ResourceType resourceType = (ResourceType) paramsMap.get(Constants.STRING_PLUGIN_PARAM_TYPE); return resourceType != null && resourceType.equals(ResourceType.UDF); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java index 7d76d5c4bd..c3a9a845f5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java @@ -17,18 +17,16 @@ package org.apache.dolphinscheduler.api.audit.operator; -import org.apache.dolphinscheduler.api.audit.OperatorLog; -import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import java.util.List; import java.util.Map; -import io.swagger.v3.oas.annotations.Operation; - public interface AuditOperator { - void recordAudit(Map paramsMap, - Result result, - long latency, - Operation operation, - OperatorLog operatorLog) throws Throwable; + void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue); + + void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java index 270a5e4df4..0ab607da65 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.api.audit.operator; import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; import org.apache.dolphinscheduler.api.audit.OperatorUtils; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.service.AuditService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.commons.lang3.math.NumberUtils; @@ -36,7 +36,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.base.Strings; -import io.swagger.v3.oas.annotations.Operation; @Service @Slf4j @@ -46,40 +45,34 @@ public abstract class BaseAuditOperator implements AuditOperator { private AuditService auditService; @Override - public void recordAudit(Map paramsMap, - Result result, - long latency, - Operation operation, - OperatorLog operatorLog) { + public void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue) { + Result result = new Result<>(); + if (returnValue instanceof Result) { + result = (Result) returnValue; + if (OperatorUtils.resultFail(result)) { + log.error("request fail, code {}", result.getCode()); + return; + } + } + + long latency = System.currentTimeMillis() - auditContext.getBeginTime(); + List auditLogList = auditContext.getAuditLogList(); + + Map paramsMap = auditContext.getParamsMap(); + OperatorLog operatorLog = auditContext.getOperatorLog(); AuditType auditType = operatorLog.auditType(); - User user = OperatorUtils.getUser(paramsMap); - - if (user == null) { - log.error("user is null"); - return; - } - - List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); - setRequestParam(auditType, auditLogList, paramsMap); - - if (OperatorUtils.resultFail(result)) { - log.error("request fail, code {}", result.getCode()); - return; - } - setObjectIdentityFromReturnObject(auditType, result, auditLogList); - modifyAuditOperationType(auditType, paramsMap, auditLogList); modifyAuditObjectType(auditType, paramsMap, auditLogList); auditLogList.forEach(auditLog -> auditLog.setLatency(latency)); auditLogList.forEach(auditLog -> auditService.addAudit(auditLog)); - } - protected void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { + @Override + public void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { String[] paramNameArr = auditType.getRequestParamName(); if (paramNameArr.length == 0) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 8b01df1319..71e4d04d6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -105,8 +105,6 @@ import com.google.common.collect.Lists; @Slf4j public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDefinitionService { - private static final String RELEASESTATE = "releaseState"; - @Autowired private ProjectMapper projectMapper; @@ -1297,7 +1295,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } if (null == releaseState) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(code); @@ -1337,7 +1335,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe break; default: log.warn("Parameter releaseState is invalid."); - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } int update = taskDefinitionMapper.updateById(taskDefinition); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 19e1a1fabb..b5bbf740e9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -747,4 +747,8 @@ public final class Constants { * K8S sensitive param */ public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : \"))).*?(?=(\",\\n))"; + + public static final String RELEASE_STATE = "releaseState"; + public static final String EXECUTE_TYPE = "executeType"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java index 449f046f4a..5f046882c1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java @@ -29,7 +29,7 @@ import lombok.Getter; @Getter public enum AuditModelType { - PROJECT("Project", null), // 1 + PROJECT("Project", null), PROCESS("Process", PROJECT), PROCESS_INSTANCE("ProcessInstance", PROCESS), TASK("Task", PROCESS), From a9decc911f8141b7dbc1de8b4f0599e0e170a1ff Mon Sep 17 00:00:00 2001 From: Gallardot Date: Fri, 19 Apr 2024 16:52:17 +0800 Subject: [PATCH 23/41] [Bug][Helm] fix image.registry (#15860) Signed-off-by: Gallardot Co-authored-by: fuchanghai Co-authored-by: Rick Cheng --- deploy/kubernetes/dolphinscheduler/README.md | 2 +- .../kubernetes/dolphinscheduler/values.yaml | 2 +- docs/docs/en/guide/installation/kubernetes.md | 19 +++++++++---------- docs/docs/zh/guide/installation/kubernetes.md | 19 +++++++++---------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 33633f3b2e..ba533b6e47 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -174,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | image.master | string | `"dolphinscheduler-master"` | master image | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent | | image.pullSecret | string | `""` | Specify a imagePullSecrets | -| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler | +| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler | | image.tag | string | `"latest"` | Docker image version for the DolphinScheduler | | image.tools | string | `"dolphinscheduler-tools"` | tools image | | image.worker | string | `"dolphinscheduler-worker"` | worker image | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 98c2f70db0..7a04ff5604 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -31,7 +31,7 @@ initImage: image: # -- Docker image repository for the DolphinScheduler - registry: apache/dolphinscheduler + registry: apache # -- Docker image version for the DolphinScheduler tag: latest # -- Image pull policy. Options: Always, Never, IfNotPresent diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index 8e58fdeccd..a6587a5855 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco ## Install DolphinScheduler -Please download the source code package `apache-dolphinscheduler--src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download) - -To publish the release name `dolphinscheduler` version, please execute the following commands: - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# Choose the corresponding version yourself +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` To publish the release name `dolphinscheduler` version to `test` namespace: diff --git a/docs/docs/zh/guide/installation/kubernetes.md b/docs/docs/zh/guide/installation/kubernetes.md index 20cd8907e8..f4b95de27b 100644 --- a/docs/docs/zh/guide/installation/kubernetes.md +++ b/docs/docs/zh/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务 ## 安装 dolphinscheduler -请下载源码包 apache-dolphinscheduler--src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download) - -发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令： - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# 自行选择对应的版本 +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` 将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中： From 285c5a8eb541f46b6e7b508729ba4b5178d09152 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 19 Apr 2024 18:12:40 +0800 Subject: [PATCH 24/41] [DSIP-28] Donnot scan whole bean under classpath (#15874) --- .../dolphinscheduler/alert/AlertServer.java | 13 +- .../api/ApiApplicationServer.java | 26 +- .../api/dto/task/TaskUpdateRequest.java | 5 +- .../impl/ProcessDefinitionServiceImpl.java | 7 +- .../impl/ProcessInstanceServiceImpl.java | 13 +- .../impl/TaskDefinitionServiceImpl.java | 75 ++- .../service/ProcessInstanceServiceTest.java | 36 +- .../TaskDefinitionServiceImplTest.java | 460 +++++++++--------- .../src/test/resources/logback-spring.xml | 57 +++ .../common/CommonConfiguration.java | 26 + .../common/log/remote/RemoteLogUtils.java | 1 - .../common/process/HttpProperty.java | 124 ----- .../api/DatabaseEnvironmentCondition.java | 39 ++ ...java => H2DaoPluginAutoConfiguration.java} | 8 +- .../h2/H2DatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + ...a => MysqlDaoPluginAutoConfiguration.java} | 8 +- .../MysqlDatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + ...PostgresqlDaoPluginAutoConfiguration.java} | 8 +- ...ostgresqlDatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + .../dao/DaoConfiguration.java | 2 +- .../dolphinscheduler/dao/BaseDaoTest.java | 2 - .../dao/repository/impl/AlertDaoTest.java | 2 - .../server/master/MasterServer.java | 28 +- .../runner/TaskExecutionContextFactory.java | 7 +- .../src/test/resources/logback.xml | 2 +- ...ation.java => MeterAutoConfiguration.java} | 13 +- .../meter/metrics/DefaultMetricsProvider.java | 3 - .../main/resources/META-INF/spring.factories | 19 + .../registry/api/RegistryConfiguration.java | 33 ++ .../plugin/registry/etcd/EtcdRegistry.java | 12 +- .../etcd/EtcdRegistryAutoConfiguration.java | 48 ++ .../registry/etcd/EtcdRegistryProperties.java | 2 - .../main/resources/META-INF/spring.factories | 19 + .../jdbc/JdbcRegistryAutoConfiguration.java | 2 + .../registry/zookeeper/ZookeeperRegistry.java | 7 +- .../ZookeeperRegistryAutoConfiguration.java | 46 ++ .../ZookeeperRegistryProperties.java | 94 +--- .../main/resources/META-INF/spring.factories | 19 + ... => QuartzSchedulerAutoConfiguration.java} | 2 +- .../main/resources/META-INF/spring.factories | 19 + .../service/ServiceConfiguration.java | 26 + .../service/process/ProcessServiceImpl.java | 4 - .../service/utils/CommonUtils.java | 108 ---- .../service/process/ProcessServiceTest.java | 5 - .../service/utils/CommonUtilsTest.java | 71 --- .../dolphinscheduler/StandaloneServer.java | 2 +- .../src/main/resources/application.yaml | 2 +- .../plugin/task/api/TaskPluginManager.java | 27 +- .../datasource/UpgradeDolphinScheduler.java | 4 +- .../server/worker/WorkerServer.java | 22 +- .../runner/DefaultWorkerTaskExecutor.java | 3 - .../DefaultWorkerTaskExecutorFactory.java | 5 - .../worker/runner/WorkerTaskExecutor.java | 5 +- .../WorkerTaskExecutorFactoryBuilder.java | 11 - .../runner/DefaultWorkerTaskExecutorTest.java | 5 - .../WorkerTaskExecutorThreadPoolTest.java | 3 +- .../TaskInstanceOperationFunctionTest.java | 5 - .../src/test/resources/logback.xml | 14 +- 61 files changed, 895 insertions(+), 855 deletions(-) create mode 100644 dolphinscheduler-api/src/test/resources/logback-spring.xml create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/{H2DaoPluginConfiguration.java => H2DaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/{MysqlDaoPluginConfiguration.java => MysqlDaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/{PostgresqlDaoPluginConfiguration.java => PostgresqlDaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories rename dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/{MeterConfiguration.java => MeterAutoConfiguration.java} (87%) create mode 100644 dolphinscheduler-meter/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories rename dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/{QuartzSchedulerConfiguration.java => QuartzSchedulerAutoConfiguration.java} (97%) create mode 100644 dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java delete mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java delete mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ff0088ea8d..55c5c3446c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -23,11 +23,13 @@ import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -37,14 +39,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; @Slf4j +@Import({CommonConfiguration.class, + DaoConfiguration.class, + RegistryConfiguration.class}) @SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) public class AlertServer { @Autowired diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 20a6412076..6f7d8f43d2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -18,13 +18,17 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; +import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -38,21 +42,19 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.servlet.ServletComponentScan; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; import org.springframework.context.event.EventListener; +@Slf4j +@Import({DaoConfiguration.class, + CommonConfiguration.class, + ServiceConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) @ServletComponentScan @SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) -@Slf4j public class ApiApplicationServer { - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private PluginDao pluginDao; @@ -66,8 +68,8 @@ public class ApiApplicationServer { public void run(ApplicationReadyEvent readyEvent) { log.info("Received spring application context ready event will load taskPlugin and write to DB"); // install task plugin - taskPluginManager.loadPlugin(); - for (Map.Entry entry : taskPluginManager.getTaskChannelFactoryMap().entrySet()) { + TaskPluginManager.loadPlugin(); + for (Map.Entry entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { String taskPluginName = entry.getKey(); TaskChannelFactory taskChannelFactory = entry.getValue(); List params = taskChannelFactory.getParams(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java index b5a8ea7a46..d7b026ed16 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java @@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.commons.beanutils.BeanUtils; -import java.lang.reflect.InvocationTargetException; import java.util.Date; import lombok.Data; +import lombok.SneakyThrows; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -107,7 +107,8 @@ public class TaskUpdateRequest { * @param taskDefinition exists task definition object * @return task definition */ - public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) throws InvocationTargetException, IllegalAccessException, InstantiationException, NoSuchMethodException { + @SneakyThrows + public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) { TaskDefinition taskDefinitionDeepCopy = (TaskDefinition) BeanUtils.cloneBean(taskDefinition); assert taskDefinitionDeepCopy != null; if (this.name != null) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 2bf84a0bcc..e6893b04fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -238,9 +238,6 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro @Autowired private DataSourceMapper dataSourceMapper; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private WorkFlowLineageService workFlowLineageService; @@ -424,7 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson); } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -1618,7 +1615,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check whether the process definition json is normal for (TaskNode taskNode : taskNodes) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskNode.getType()) .taskParams(taskNode.getTaskParams()) .dependence(taskNode.getDependence()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 36cc986607..06e9fd95db 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -70,11 +70,9 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.RelationSubWorkflowMapper; -import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; @@ -177,18 +175,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce @Autowired UsersService usersService; - @Autowired - private TenantMapper tenantMapper; - @Autowired TaskDefinitionMapper taskDefinitionMapper; - @Autowired - private TaskPluginManager taskPluginManager; - - @Autowired - private ScheduleMapper scheduleMapper; - @Autowired private RelationSubWorkflowMapper relationSubWorkflowMapper; @@ -725,7 +714,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 71e4d04d6f..2887606a1d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -75,7 +75,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -135,9 +134,6 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe @Autowired private ProcessService processService; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private ProcessDefinitionService processDefinitionService; @@ -147,8 +143,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create task definition * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param taskDefinitionJson task definition json */ @Transactional @@ -171,7 +167,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -212,7 +208,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe Project project = projectMapper.queryByCode(taskDefinition.getProjectCode()); projectService.checkProjectAndAuthThrowException(user, project, permissions); - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -242,7 +238,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Create resource task definition * - * @param loginUser login user + * @param loginUser login user * @param taskCreateRequest task definition json * @return new TaskDefinition have created */ @@ -286,11 +282,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create single task definition that binds the workflow * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param processDefinitionCode process definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return create result code */ @Transactional @@ -325,7 +321,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return result; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -412,10 +408,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * query task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code * @param processCode process code - * @param taskName task name + * @param taskName task name */ @Override public Map queryTaskDefinitionByName(User loginUser, long projectCode, long processCode, @@ -473,12 +469,12 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Delete resource task definition by code - * + *

* Only task release state offline and no downstream tasks can be deleted, will also remove the exists * task relation [upstreamTaskCode, taskCode] * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code */ @Transactional @Override @@ -546,9 +542,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task code * @param taskDefinitionJsonObj task definition json object */ @Transactional @@ -604,8 +600,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param taskCode task code + * @param loginUser login user + * @param taskCode task code * @param taskUpdateRequest task definition json object * @return new TaskDefinition have updated */ @@ -619,13 +615,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe throw new ServiceException(Status.TASK_DEFINITION_NOT_EXISTS, taskCode); } - TaskDefinition taskDefinitionUpdate; - try { - taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); - } catch (InvocationTargetException | IllegalAccessException | InstantiationException - | NoSuchMethodException e) { - throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, taskUpdateRequest.toString()); - } + TaskDefinition taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); + this.checkTaskDefinitionValid(loginUser, taskDefinitionUpdate, TASK_DEFINITION_UPDATE); this.TaskDefinitionUpdateValid(taskDefinitionOriginal, taskDefinitionUpdate); @@ -656,7 +647,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe * Get resource task definition by code * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code * @return TaskDefinition */ @Override @@ -674,7 +665,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Get resource task definition according to query parameter * - * @param loginUser login user + * @param loginUser login user * @param taskFilterRequest taskFilterRequest object you want to filter the resource task definitions * @return TaskDefinitions of page */ @@ -741,7 +732,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return null; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionToUpdate.getTaskType()) .taskParams(taskDefinitionToUpdate.getTaskParams()) .dependence(taskDefinitionToUpdate.getDependence()) @@ -846,11 +837,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition and upstream * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task definition code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return update result code */ @Override @@ -1019,10 +1010,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * switch task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code - * @param taskCode task code - * @param version the version user want to switch + * @param taskCode task code + * @param version the version user want to switch */ @Transactional @Override @@ -1277,9 +1268,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * release task definition * - * @param loginUser login user - * @param projectCode project code - * @param code task definition code + * @param loginUser login user + * @param projectCode project code + * @param code task definition code * @param releaseState releaseState * @return update result code */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 9fb4d83052..7d9a4f9338 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -84,6 +84,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -142,9 +143,6 @@ public class ProcessInstanceServiceTest { @Mock TaskDefinitionMapper taskDefinitionMapper; - @Mock - TaskPluginManager taskPluginManager; - @Mock ScheduleMapper scheduleMapper; @@ -625,21 +623,27 @@ public class ProcessInstanceServiceTest { List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result); putMsg(result, Status.SUCCESS, projectCode); - when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Map processInstanceFinishRes = - processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - // success - when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); - putMsg(result, Status.SUCCESS, projectCode); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Map processInstanceFinishRes = + processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) - .thenReturn(1); - Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + // success + when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); + putMsg(result, Status.SUCCESS, projectCode); + + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) + .thenReturn(1); + Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java index e77eab8029..ff8ca6ba2c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java @@ -17,13 +17,19 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_UPDATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest; import org.apache.dolphinscheduler.api.dto.task.TaskUpdateRequest; @@ -75,13 +81,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class TaskDefinitionServiceImplTest { @InjectMocks @@ -114,9 +124,6 @@ public class TaskDefinitionServiceImplTest { @Mock private ProcessTaskRelationMapper processTaskRelationMapper; - @Mock - private TaskPluginManager taskPluginManager; - @Mock private ProcessTaskRelationService processTaskRelationService; @@ -155,61 +162,73 @@ public class TaskDefinitionServiceImplTest { @Test public void createTaskDefinition() { - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, result)) - .thenReturn(true); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); + Map result = new HashMap<>(); + when(projectService.hasProjectAndWritePerm(user, project, result)) + .thenReturn(true); - String createTaskDefinitionJson = - "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" - + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," - + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" - + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," - + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," - + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," - + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; - Map relation = taskDefinitionService - .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + String createTaskDefinitionJson = + "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" + + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," + + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" + + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," + + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," + + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," + + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; + Map relation = taskDefinitionService + .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + + } } @Test public void updateTaskDefinition() { - String taskDefinitionJson = getTaskDefinitionJson();; + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS, PROJECT_CODE); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); - Mockito.when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); - Mockito.when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) - .thenReturn(getProcessTaskRelationList2()); - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); - result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - // failure - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson)); - Assertions.assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), - ((ServiceException) exception).getCode()); + when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); + when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); + when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); + when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); + when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) + .thenReturn(getProcessTaskRelationList2()); + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); + result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); + assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + // failure + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); + exception = Assertions.assertThrows(ServiceException.class, + () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson)); + assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), + ((ServiceException) exception).getCode()); + } } @@ -217,72 +236,72 @@ public class TaskDefinitionServiceImplTest { public void queryTaskDefinitionByName() { String taskName = "task"; Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(result); - Mockito.when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) + when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) .thenReturn(new TaskDefinition()); Map relation = taskDefinitionService .queryTaskDefinitionByName(user, PROJECT_CODE, PROCESS_DEFINITION_CODE, taskName); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } @Test public void deleteTaskDefinitionByCode() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); // error task definition not find exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error delete single task definition object - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), + assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_DELETE); - Mockito.when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); Assertions.assertDoesNotThrow(() -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); } @Test public void switchVersion() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when( + when( projectService.checkProjectAndAuth(user, project, PROJECT_CODE, WORKFLOW_SWITCH_TO_THIS_VERSION)) - .thenReturn(result); + .thenReturn(result); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) .thenReturn(new TaskDefinitionLog()); TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(PROJECT_CODE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)) + when(taskDefinitionMapper.queryByCode(TASK_CODE)) .thenReturn(taskDefinition); - Mockito.when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); + when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); Map relation = taskDefinitionService .switchVersion(user, PROJECT_CODE, TASK_CODE, VERSION); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } private void putMsg(Map result, Status status, Object... statusParams) { @@ -331,7 +350,7 @@ public class TaskDefinitionServiceImplTest { @Test public void genTaskCodeList() { Map genTaskCodeList = taskDefinitionService.genTaskCodeList(10); - Assertions.assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); } @Test @@ -348,31 +367,31 @@ public class TaskDefinitionServiceImplTest { taskMainInfo.setUpstreamTaskName("4"); taskMainInfoIPage.setRecords(Collections.singletonList(taskMainInfo)); taskMainInfoIPage.setTotal(10L); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(checkResult); - Mockito.when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), + when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), Mockito.isNull(), Mockito.anyString(), Mockito.isNull())) - .thenReturn(taskMainInfoIPage); - Mockito.when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) + .thenReturn(taskMainInfoIPage); + when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) .thenReturn(Collections.singletonList(taskMainInfo)); Result result = taskDefinitionService.queryTaskDefinitionListPaging(user, PROJECT_CODE, null, null, null, pageNo, pageSize); - Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test public void testReleaseTaskDefinition() { - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); Project project = getProject(); // check task dose not exist Map result = new HashMap<>(); putMsg(result, Status.TASK_DEFINE_NOT_EXIST, TASK_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); Map map = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); + assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); // process definition offline putMsg(result, Status.SUCCESS); @@ -384,23 +403,23 @@ public class TaskDefinitionServiceImplTest { "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}}"; taskDefinition.setTaskParams(params); taskDefinition.setTaskType("SHELL"); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(taskDefinition); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) .thenReturn(taskDefinitionLog); Map offlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); // process definition online, resource exist Map onlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.ONLINE); - Assertions.assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); // release error code Map failResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.getEnum(2)); - Assertions.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); + assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); } @Test @@ -410,133 +429,127 @@ public class TaskDefinitionServiceImplTest { taskCreateRequest.setWorkflowCode(PROCESS_DEFINITION_CODE); // error process definition not find - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROCESS_DEFINE_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition taskCreateRequest.setTaskParams(TASK_PARAMETER); - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); - // error create task definition object - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); - // error sync to task definition log - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error create task definition object + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Mockito.when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), - isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); - Assertions.assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + // error sync to task definition log + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), + isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); + assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + } } @Test public void testUpdateTaskDefinitionV2() { TaskUpdateRequest taskUpdateRequest = new TaskUpdateRequest(); + TaskDefinition taskDefinition = getTaskDefinition(); + Project project = getProject(); // error task definition not exists - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_EXISTS, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_EXISTS.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(isA(Long.class))).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) - .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(projectMapper.queryByCode(isA(Long.class))).thenReturn(project); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + .checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); - // error task definition already online - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(false); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + } - // error task definition nothing update - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_CHANGE.getCode(), ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + // error task definition nothing update + when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_CHANGE, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition version invalid - taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.DATA_IS_NOT_VALID.getCode(), ((ServiceException) exception).getCode()); + // error task definition version invalid + taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); + assertThrowsServiceException(Status.DATA_IS_NOT_VALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition update effect number - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.UPDATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition update effect number + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.UPDATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition log insert - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition log insert + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Assertions.assertDoesNotThrow( - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + Assertions.assertDoesNotThrow( + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - TaskDefinition taskDefinition = - taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); - Assertions.assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + taskDefinition = + taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); + assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + } } @Test @@ -549,28 +562,28 @@ public class TaskDefinitionServiceImplTest { ArrayList taskDefinitionLogs = new ArrayList<>(); taskDefinitionLogs.add(taskDefinitionLog); Integer version = 1; - Mockito.when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); + when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); // saveProcessDefine - Mockito.when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); - Mockito.when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); + when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); int insertVersion = processServiceImpl.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE); - Mockito.when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) .thenReturn(insertVersion); - Assertions.assertEquals(insertVersion, version + 1); + assertEquals(insertVersion, version + 1); // saveTaskRelation List processTaskRelationLogList = getProcessTaskRelationLogList(); - Mockito.when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) + when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) .thenReturn(processTaskRelationList); - Mockito.when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); - Mockito.when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); int insertResult = processServiceImpl.saveTaskRelation(loginUser, processDefinition.getProjectCode(), processDefinition.getCode(), insertVersion, processTaskRelationLogList, taskDefinitionLogs, Boolean.TRUE); - Assertions.assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); + assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); Assertions.assertDoesNotThrow( () -> taskDefinitionService.updateDag(loginUser, processDefinition.getCode(), processTaskRelationList, taskDefinitionLogs)); @@ -581,55 +594,60 @@ public class TaskDefinitionServiceImplTest { // error task definition not exists exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition not exists - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), + assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); Assertions.assertDoesNotThrow(() -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); } @Test public void testUpdateTaskWithUpstream() { + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); + TaskDefinition taskDefinition = getTaskDefinition(); + taskDefinition.setFlag(Flag.NO); + TaskDefinition taskDefinitionSecond = getTaskDefinition(); + taskDefinitionSecond.setCode(5); - String taskDefinitionJson = getTaskDefinitionJson(); - TaskDefinition taskDefinition = getTaskDefinition(); - taskDefinition.setFlag(Flag.NO); - TaskDefinition taskDefinitionSecond = getTaskDefinition(); - taskDefinitionSecond.setCode(5); + user.setUserType(UserType.ADMIN_USER); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); - user.setUserType(UserType.ADMIN_USER); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); + when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) + .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - Mockito.when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) - .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - - Mockito.when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) - .thenReturn(getProcessTaskRelationListV2()); - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); - Mockito.when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); - // success - Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, - taskDefinitionJson, UPSTREAM_CODE); - Assertions.assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); - user.setUserType(UserType.GENERAL_USER); + when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) + .thenReturn(getProcessTaskRelationListV2()); + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)) + .thenReturn(getProcessDefinition()); + when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); + when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); + when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); + // success + Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson, UPSTREAM_CODE); + assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); + user.setUserType(UserType.GENERAL_USER); + } } private String getTaskDefinitionJson() { diff --git a/dolphinscheduler-api/src/test/resources/logback-spring.xml b/dolphinscheduler-api/src/test/resources/logback-spring.xml new file mode 100644 index 0000000000..9159c3b02d --- /dev/null +++ b/dolphinscheduler-api/src/test/resources/logback-spring.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-api.log + + ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + 50GB + true + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java new file mode 100644 index 0000000000..5411e4cbc1 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ComponentScan("org.apache.dolphinscheduler.common") +public class CommonConfiguration { +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java index 25d8024474..8aab70f304 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java @@ -40,7 +40,6 @@ public class RemoteLogUtils { @Autowired private RemoteLogService autowiredRemoteLogService; - @PostConstruct private void init() { remoteLogService = autowiredRemoteLogService; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java deleted file mode 100644 index 11786fd5a3..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.process; - -import org.apache.dolphinscheduler.common.enums.HttpParametersType; - -import java.util.Objects; - -public class HttpProperty { - - /** - * key - */ - private String prop; - - /** - * httpParametersType - */ - private HttpParametersType httpParametersType; - - /** - * value - */ - private String value; - - public HttpProperty() { - } - - public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { - this.prop = prop; - this.httpParametersType = httpParametersType; - this.value = value; - } - - /** - * getter method - * - * @return the prop - * @see HttpProperty#prop - */ - public String getProp() { - return prop; - } - - /** - * setter method - * - * @param prop the prop to set - * @see HttpProperty#prop - */ - public void setProp(String prop) { - this.prop = prop; - } - - /** - * getter method - * - * @return the value - * @see HttpProperty#value - */ - public String getValue() { - return value; - } - - /** - * setter method - * - * @param value the value to set - * @see HttpProperty#value - */ - public void setValue(String value) { - this.value = value; - } - - public HttpParametersType getHttpParametersType() { - return httpParametersType; - } - - public void setHttpParametersType(HttpParametersType httpParametersType) { - this.httpParametersType = httpParametersType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - HttpProperty property = (HttpProperty) o; - return Objects.equals(prop, property.prop) - && Objects.equals(value, property.value); - } - - @Override - public int hashCode() { - return Objects.hash(prop, value); - } - - @Override - public String toString() { - return "HttpProperty{" - + "prop='" + prop + '\'' - + ", httpParametersType=" + httpParametersType - + ", value='" + value + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..32fb807a9b --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.api; + +import java.util.Arrays; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class DatabaseEnvironmentCondition implements Condition { + + private final String profile; + + public DatabaseEnvironmentCondition(String profile) { + this.profile = profile; + } + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + String[] activeProfiles = context.getEnvironment().getActiveProfiles(); + return Arrays.asList(activeProfiles).contains(profile); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java index 9aea94f77d..f496679ed5 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("h2") -@Configuration -public class H2DaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(H2DatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class H2DaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..894f38bd20 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.h2; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class H2DatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public H2DatabaseEnvironmentCondition() { + super("h2"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..c899dfb43f --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.h2.H2DaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java index 8b37fca67b..5fb3a350ae 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java @@ -28,14 +28,14 @@ import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("mysql") -@Configuration -public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Configuration(proxyBeanMethods = false) +@Conditional(MysqlDatabaseEnvironmentCondition.class) +public class MysqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2136e1354e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.mysql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class MysqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public MysqlDatabaseEnvironmentCondition() { + super("mysql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..386c80c676 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.mysql.MysqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java index e57c84fab9..f0467bfd07 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonit import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("postgresql") -@Configuration -public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(PostgresqlDatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class PostgresqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2c71ea8442 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.postgresql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class PostgresqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public PostgresqlDatabaseEnvironmentCondition() { + super("postgresql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..fd6a5f07b9 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.postgresql.PostgresqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index 985f5b56b4..e089c8086b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -40,8 +40,8 @@ import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; @Configuration +@ComponentScan("org.apache.dolphinscheduler.dao") @EnableAutoConfiguration -@ComponentScan({"org.apache.dolphinscheduler.dao.plugin"}) @MapperScan(basePackages = "org.apache.dolphinscheduler.dao.mapper", sqlSessionFactoryRef = "sqlSessionFactory") public class DaoConfiguration { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java index 231c947f65..6f14eb436e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java @@ -22,7 +22,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; -import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; @ExtendWith(MockitoExtension.class) @@ -30,6 +29,5 @@ import org.springframework.transaction.annotation.Transactional; @SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) @Transactional @Rollback -@EnableTransactionManagement public abstract class BaseDaoTest { } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index c0a841c1a0..fe8545f3e2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -34,7 +34,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ActiveProfiles; -import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; @ActiveProfiles(ProfileType.H2) @@ -43,7 +42,6 @@ import org.springframework.transaction.annotation.Transactional; @SpringBootTest(classes = DaoConfiguration.class) @Transactional @Rollback -@EnableTransactionManagement public class AlertDaoTest { @Autowired diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index fd262a8c6e..7988570135 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -17,15 +17,18 @@ package org.apache.dolphinscheduler.server.master; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.scheduler.api.SchedulerApi; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient; @@ -35,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.EventExecuteService; import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap; import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator; +import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import javax.annotation.PostConstruct; @@ -45,18 +49,15 @@ import org.quartz.SchedulerException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; -@SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) -@EnableTransactionManagement -@EnableCaching @Slf4j +@Import({DaoConfiguration.class, + ServiceConfiguration.class, + CommonConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) +@SpringBootApplication public class MasterServer implements IStoppable { @Autowired @@ -65,9 +66,6 @@ public class MasterServer implements IStoppable { @Autowired private MasterRegistryClient masterRegistryClient; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private MasterSchedulerBootstrap masterSchedulerBootstrap; @@ -109,7 +107,7 @@ public class MasterServer implements IStoppable { this.masterRPCServer.start(); // install task plugin - this.taskPluginManager.loadPlugin(); + TaskPluginManager.loadPlugin(); this.masterSlotManager.start(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index ab1806ff67..b3b0cf67ea 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -90,9 +90,6 @@ public class TaskExecutionContextFactory { @Autowired private ProcessService processService; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private CuringParamsService curingParamsService; @@ -106,14 +103,14 @@ public class TaskExecutionContextFactory { ProcessInstance workflowInstance = taskInstance.getProcessInstance(); ResourceParametersHelper resources = - Optional.ofNullable(taskPluginManager.getTaskChannel(taskInstance.getTaskType())) + Optional.ofNullable(TaskPluginManager.getTaskChannel(taskInstance.getTaskType())) .map(taskChannel -> taskChannel.getResources(taskInstance.getTaskParams())) .orElse(null); setTaskResourceInfo(resources); Map businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance); - AbstractParameters baseParam = taskPluginManager.getParameters(ParametersNode.builder() + AbstractParameters baseParam = TaskPluginManager.getParameters(ParametersNode.builder() .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); Map propertyMap = curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance); diff --git a/dolphinscheduler-master/src/test/resources/logback.xml b/dolphinscheduler-master/src/test/resources/logback.xml index 4470a46391..286e35cd1f 100644 --- a/dolphinscheduler-master/src/test/resources/logback.xml +++ b/dolphinscheduler-master/src/test/resources/logback.xml @@ -65,7 +65,7 @@ - + diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java similarity index 87% rename from dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java rename to dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java index e3b140a578..fa8933d132 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java @@ -20,7 +20,8 @@ package org.apache.dolphinscheduler.meter; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.apache.dolphinscheduler.meter.metrics.DefaultMetricsProvider; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -42,11 +43,15 @@ import io.micrometer.core.instrument.MeterRegistry; * } * */ -@Configuration +@Configuration(proxyBeanMethods = false) @EnableAspectJAutoProxy -@EnableAutoConfiguration @ConditionalOnProperty(prefix = "metrics", name = "enabled", havingValue = "true") -public class MeterConfiguration { +public class MeterAutoConfiguration { + + @Bean + public DefaultMetricsProvider metricsProvider(MeterRegistry meterRegistry) { + return new DefaultMetricsProvider(meterRegistry); + } @Bean public TimedAspect timedAspect(MeterRegistry registry) { diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java index f1240a0541..e293e44a8d 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java @@ -19,11 +19,8 @@ package org.apache.dolphinscheduler.meter.metrics; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.springframework.stereotype.Component; - import io.micrometer.core.instrument.MeterRegistry; -@Component public class DefaultMetricsProvider implements MetricsProvider { private final MeterRegistry meterRegistry; diff --git a/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories b/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..77bc56d86e --- /dev/null +++ b/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.meter.MeterAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java new file mode 100644 index 0000000000..bae3694964 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RegistryConfiguration { + + @Bean + @ConditionalOnMissingBean + public RegistryClient registryClient(Registry registry) { + return new RegistryClient(registry); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 1d1397db54..6833a6607b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -41,8 +41,6 @@ import javax.net.ssl.SSLException; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import com.google.common.base.Splitter; @@ -68,8 +66,6 @@ import io.netty.handler.ssl.SslContext; * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to * store the DolphinScheduler master/worker's metadata and do the server registry/unRegistry. */ -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @Slf4j public class EtcdRegistry implements Registry { @@ -86,6 +82,7 @@ public class EtcdRegistry implements Registry { private final Map watcherMap = new ConcurrentHashMap<>(); private static final long TIME_TO_LIVE_SECONDS = 30L; + public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException { ClientBuilder clientBuilder = Client.builder() .endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints()))) @@ -141,8 +138,7 @@ public class EtcdRegistry implements Registry { } /** - * - * @param path The prefix of the key being listened to + * @param path The prefix of the key being listened to * @param listener * @return if subcribe Returns true if no exception was thrown */ @@ -165,8 +161,8 @@ public class EtcdRegistry implements Registry { } /** - * @throws throws an exception if the unsubscribe path does not exist * @param path The prefix of the key being listened to + * @throws throws an exception if the unsubscribe path does not exist */ @Override public void unsubscribe(String path) { @@ -184,7 +180,6 @@ public class EtcdRegistry implements Registry { } /** - * * @return Returns the value corresponding to the key * @throws throws an exception if the key does not exist */ @@ -202,7 +197,6 @@ public class EtcdRegistry implements Registry { } /** - * * @param deleteOnDisconnect Does the put data disappear when the client disconnects */ @Override diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java new file mode 100644 index 0000000000..1038d312ff --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.etcd; + +import org.apache.dolphinscheduler.registry.api.Registry; + +import javax.net.ssl.SSLException; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@ComponentScan +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") +public class EtcdRegistryAutoConfiguration { + + public EtcdRegistryAutoConfiguration() { + log.info("Load EtcdRegistryAutoConfiguration"); + } + + @Bean + @ConditionalOnMissingBean(value = Registry.class) + public EtcdRegistry etcdRegistry(EtcdRegistryProperties etcdRegistryProperties) throws SSLException { + return new EtcdRegistry(etcdRegistryProperties); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index faded2a506..babb6dea76 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -21,13 +21,11 @@ import java.time.Duration; import lombok.Data; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @Data @Configuration -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @ConfigurationProperties(prefix = "registry") public class EtcdRegistryProperties { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..689817bb96 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.etcd.EtcdRegistryAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java index 09211f99fb..f21ce0d67c 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java @@ -30,6 +30,7 @@ import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration; @@ -37,6 +38,7 @@ import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import com.zaxxer.hikari.HikariDataSource; @Slf4j +@ComponentScan @Configuration(proxyBeanMethods = false) @MapperScan("org.apache.dolphinscheduler.plugin.registry.jdbc.mapper") @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 7333c10f05..3f0c3ccb59 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -50,14 +50,11 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; import lombok.NonNull; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; +import lombok.extern.slf4j.Slf4j; import com.google.common.base.Strings; -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") +@Slf4j public final class ZookeeperRegistry implements Registry { private final ZookeeperRegistryProperties.ZookeeperProperties properties; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java new file mode 100644 index 0000000000..e8fc31327e --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.zookeeper; + +import org.apache.dolphinscheduler.registry.api.Registry; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@ComponentScan +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") +public class ZookeeperRegistryAutoConfiguration { + + public ZookeeperRegistryAutoConfiguration() { + log.info("Load ZookeeperRegistryAutoConfiguration"); + } + + @Bean + @ConditionalOnMissingBean(value = Registry.class) + public ZookeeperRegistry zookeeperRegistry(ZookeeperRegistryProperties zookeeperRegistryProperties) { + return new ZookeeperRegistry(zookeeperRegistryProperties); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java index 42dbc5d256..b54ebc0b1a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java @@ -19,25 +19,19 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper; import java.time.Duration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +@Data @Configuration -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") @ConfigurationProperties(prefix = "registry") public class ZookeeperRegistryProperties { private ZookeeperProperties zookeeper = new ZookeeperProperties(); - public ZookeeperProperties getZookeeper() { - return zookeeper; - } - - public void setZookeeper(ZookeeperProperties zookeeper) { - this.zookeeper = zookeeper; - } - + @Data public static final class ZookeeperProperties { private String namespace; @@ -48,91 +42,13 @@ public class ZookeeperRegistryProperties { private Duration connectionTimeout = Duration.ofSeconds(9); private Duration blockUntilConnected = Duration.ofMillis(600); - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getConnectString() { - return connectString; - } - - public void setConnectString(String connectString) { - this.connectString = connectString; - } - - public RetryPolicy getRetryPolicy() { - return retryPolicy; - } - - public void setRetryPolicy(RetryPolicy retryPolicy) { - this.retryPolicy = retryPolicy; - } - - public String getDigest() { - return digest; - } - - public void setDigest(String digest) { - this.digest = digest; - } - - public Duration getSessionTimeout() { - return sessionTimeout; - } - - public void setSessionTimeout(Duration sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } - - public Duration getConnectionTimeout() { - return connectionTimeout; - } - - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Duration getBlockUntilConnected() { - return blockUntilConnected; - } - - public void setBlockUntilConnected(Duration blockUntilConnected) { - this.blockUntilConnected = blockUntilConnected; - } - + @Data public static final class RetryPolicy { private Duration baseSleepTime = Duration.ofMillis(60); private int maxRetries; private Duration maxSleep = Duration.ofMillis(300); - public Duration getBaseSleepTime() { - return baseSleepTime; - } - - public void setBaseSleepTime(Duration baseSleepTime) { - this.baseSleepTime = baseSleepTime; - } - - public int getMaxRetries() { - return maxRetries; - } - - public void setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - } - - public Duration getMaxSleep() { - return maxSleep; - } - - public void setMaxSleep(Duration maxSleep) { - this.maxSleep = maxSleep; - } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..821f1a70c1 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperRegistryAutoConfiguration diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java similarity index 97% rename from dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java rename to dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java index 07ff8af434..34fb782587 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.Bean; @AutoConfiguration(after = {QuartzAutoConfiguration.class}) @ConditionalOnClass(value = Scheduler.class) -public class QuartzSchedulerConfiguration { +public class QuartzSchedulerAutoConfiguration { @Bean @ConditionalOnMissingBean diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..b34f896b84 --- /dev/null +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.scheduler.quartz.QuartzSchedulerAutoConfiguration diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java new file mode 100644 index 0000000000..fa831a5b6b --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.service; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@ComponentScan("org.apache.dolphinscheduler.service") +@Configuration +public class ServiceConfiguration { +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 3c207ae982..635948fc86 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -111,7 +111,6 @@ import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcCli import org.apache.dolphinscheduler.extract.common.ILogService; import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState; @@ -262,9 +261,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private WorkFlowLineageMapper workFlowLineageMapper; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private ClusterMapper clusterMapper; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java deleted file mode 100644 index e00212823a..0000000000 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.utils; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; - -import java.net.URL; -import java.nio.charset.StandardCharsets; - -import lombok.extern.slf4j.Slf4j; - -/** - * common utils - */ -@Slf4j -public class CommonUtils { - - private static final Base64 BASE64 = new Base64(); - - protected CommonUtils() { - throw new UnsupportedOperationException("Construct CommonUtils"); - } - - /** - * @return get the path of system environment variables - */ - public static String getSystemEnvPath() { - String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH); - if (StringUtils.isEmpty(envPath)) { - URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH); - - if (envDefaultPath != null) { - envPath = envDefaultPath.getPath(); - log.debug("env path :{}", envPath); - } else { - envPath = "/etc/profile"; - } - } - - return envPath; - } - - /** - * encode password - */ - public static String encodePassword(String password) { - if (StringUtils.isEmpty(password)) { - return StringUtils.EMPTY; - } - // if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false); - if (!encryptionEnable) { - return password; - } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, - DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))); - return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); - } - - /** - * decode password - */ - public static String decodePassword(String password) { - if (StringUtils.isEmpty(password)) { - return StringUtils.EMPTY; - } - - // if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false); - if (!encryptionEnable) { - return password; - } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, - DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8); - if (!passwordWithSalt.startsWith(salt)) { - log.warn("There is a password and salt mismatch: {} ", password); - return password; - } - return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8); - } - -} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index 509f9a4cf1..5c998e1f9f 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -79,7 +79,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.dp.InputType; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.service.cron.CronUtilsTest; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; @@ -102,8 +101,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * process service test @@ -112,7 +109,6 @@ import org.slf4j.LoggerFactory; @MockitoSettings(strictness = Strictness.LENIENT) public class ProcessServiceTest { - private static final Logger logger = LoggerFactory.getLogger(CronUtilsTest.class); @InjectMocks private ProcessServiceImpl processService; @Mock @@ -667,7 +663,6 @@ public class ProcessServiceTest { taskDefinition.setVersion(1); taskDefinition.setCreateTime(new Date()); taskDefinition.setUpdateTime(new Date()); - when(taskPluginManager.getParameters(any())).thenReturn(null); when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskDefinition.getCode(), taskDefinition.getVersion())).thenReturn(taskDefinition); when(taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode())).thenReturn(1); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java deleted file mode 100644 index cab280a702..0000000000 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.utils; - -import org.apache.dolphinscheduler.common.utils.FileUtils; - -import java.net.InetAddress; -import java.net.UnknownHostException; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * configuration test - */ -@ExtendWith(MockitoExtension.class) -public class CommonUtilsTest { - - private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); - - @Test - public void getSystemEnvPath() { - String envPath; - envPath = CommonUtils.getSystemEnvPath(); - Assertions.assertEquals("/etc/profile", envPath); - } - - @Test - public void getDownloadFilename() { - logger.info(FileUtils.getDownloadFilename("a.txt")); - Assertions.assertTrue(true); - } - - @Test - public void getUploadFilename() { - logger.info(FileUtils.getUploadFilename("1234", "a.txt")); - Assertions.assertTrue(true); - } - - @Test - public void test() { - InetAddress ip; - try { - ip = InetAddress.getLocalHost(); - logger.info(ip.getHostAddress()); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - Assertions.assertTrue(true); - } - -} diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java index 14808ab629..a39d3c9a22 100644 --- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java +++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java @@ -24,8 +24,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication @Slf4j +@SpringBootApplication public class StandaloneServer { public static void main(String[] args) throws Exception { diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 654113471d..5122eea2a1 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -329,4 +329,4 @@ spring: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 username: root - password: root@123 + password: root diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java index be3417466f..938aa77459 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java @@ -36,21 +36,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -@Component @Slf4j public class TaskPluginManager { - private final Map taskChannelFactoryMap = new HashMap<>(); - private final Map taskChannelMap = new HashMap<>(); + private static final Map taskChannelFactoryMap = new HashMap<>(); + private static final Map taskChannelMap = new HashMap<>(); - private final AtomicBoolean loadedFlag = new AtomicBoolean(false); + private static final AtomicBoolean loadedFlag = new AtomicBoolean(false); /** * Load task plugins from classpath. */ - public void loadPlugin() { + public static void loadPlugin() { if (!loadedFlag.compareAndSet(false, true)) { log.warn("The task plugin has already been loaded"); return; @@ -70,24 +67,24 @@ public class TaskPluginManager { } - public Map getTaskChannelMap() { + public static Map getTaskChannelMap() { return Collections.unmodifiableMap(taskChannelMap); } - public Map getTaskChannelFactoryMap() { + public static Map getTaskChannelFactoryMap() { return Collections.unmodifiableMap(taskChannelFactoryMap); } - public TaskChannel getTaskChannel(String type) { - return this.getTaskChannelMap().get(type); + public static TaskChannel getTaskChannel(String type) { + return getTaskChannelMap().get(type); } - public boolean checkTaskParameters(ParametersNode parametersNode) { - AbstractParameters abstractParameters = this.getParameters(parametersNode); + public static boolean checkTaskParameters(ParametersNode parametersNode) { + AbstractParameters abstractParameters = getParameters(parametersNode); return abstractParameters != null && abstractParameters.checkParameters(); } - public AbstractParameters getParameters(ParametersNode parametersNode) { + public static AbstractParameters getParameters(ParametersNode parametersNode) { String taskType = parametersNode.getTaskType(); if (Objects.isNull(taskType)) { return null; @@ -106,7 +103,7 @@ public class TaskPluginManager { case TaskConstants.TASK_TYPE_DYNAMIC: return JSONUtils.parseObject(parametersNode.getTaskParams(), DynamicParameters.class); default: - TaskChannel taskChannel = this.getTaskChannelMap().get(taskType); + TaskChannel taskChannel = getTaskChannelMap().get(taskType); if (Objects.isNull(taskChannel)) { return null; } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java index aa9553a308..0588b20dda 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java @@ -23,12 +23,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.ImportAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; -@ImportAutoConfiguration(DaoConfiguration.class) +@Import(DaoConfiguration.class) @SpringBootApplication public class UpgradeDolphinScheduler { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index b0866f7fd8..8dd842a8ed 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -24,11 +25,12 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -47,24 +49,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; -@SpringBootApplication -@EnableTransactionManagement -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) @Slf4j +@Import({CommonConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) +@SpringBootApplication public class WorkerServer implements IStoppable { @Autowired private WorkerRegistryClient workerRegistryClient; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private WorkerRpcServer workerRpcServer; @@ -89,7 +85,7 @@ public class WorkerServer implements IStoppable { @PostConstruct public void run() { this.workerRpcServer.start(); - this.taskPluginManager.loadPlugin(); + TaskPluginManager.loadPlugin(); this.workerRegistryClient.setRegistryStoppable(this); this.workerRegistryClient.start(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java index 19421ee05e..ea44f1790f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -35,13 +34,11 @@ public class DefaultWorkerTaskExecutor extends WorkerTaskExecutor { public DefaultWorkerTaskExecutor(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java index 0141a5cd17..20fa6a5e2a 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -35,20 +34,17 @@ public class DefaultWorkerTaskExecutorFactory private final @NonNull TaskExecutionContext taskExecutionContext; private final @NonNull WorkerConfig workerConfig; private final @NonNull WorkerMessageSender workerMessageSender; - private final @NonNull TaskPluginManager taskPluginManager; private final @Nullable StorageOperate storageOperate; private final @NonNull WorkerRegistryClient workerRegistryClient; public DefaultWorkerTaskExecutorFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; } @@ -59,7 +55,6 @@ public class DefaultWorkerTaskExecutorFactory taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java index f713605a48..41cef62028 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java @@ -75,7 +75,6 @@ public abstract class WorkerTaskExecutor implements Runnable { protected final TaskExecutionContext taskExecutionContext; protected final WorkerConfig workerConfig; protected final WorkerMessageSender workerMessageSender; - protected final TaskPluginManager taskPluginManager; protected final @Nullable StorageOperate storageOperate; protected final WorkerRegistryClient workerRegistryClient; @@ -85,13 +84,11 @@ public abstract class WorkerTaskExecutor implements Runnable { @NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); @@ -220,7 +217,7 @@ public abstract class WorkerTaskExecutor implements Runnable { log.info("WorkflowInstanceExecDir: {} check successfully", taskExecutionContext.getExecutePath()); TaskChannel taskChannel = - Optional.ofNullable(taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) + Optional.ofNullable(TaskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) .orElseThrow(() -> new TaskPluginException(taskExecutionContext.getTaskType() + " task plugin not found, please check the task type is correct.")); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java index 599746818d..56f3207884 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -36,12 +35,6 @@ public class WorkerTaskExecutorFactoryBuilder { @Autowired private WorkerMessageSender workerMessageSender; - @Autowired - private TaskPluginManager taskPluginManager; - - @Autowired - private WorkerTaskExecutorThreadPool workerManager; - @Autowired(required = false) private StorageOperate storageOperate; @@ -51,14 +44,11 @@ public class WorkerTaskExecutorFactoryBuilder { public WorkerTaskExecutorFactoryBuilder( WorkerConfig workerConfig, WorkerMessageSender workerMessageSender, - TaskPluginManager taskPluginManager, WorkerTaskExecutorThreadPool workerManager, StorageOperate storageOperate, WorkerRegistryClient workerRegistryClient) { this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; - this.workerManager = workerManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; } @@ -67,7 +57,6 @@ public class WorkerTaskExecutorFactoryBuilder { return new DefaultWorkerTaskExecutorFactory(taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java index 43ef6f87d0..e211fcdd1f 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -40,8 +39,6 @@ public class DefaultWorkerTaskExecutorTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); - private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); private WorkerRegistryClient workerRegistryClient = Mockito.mock(WorkerRegistryClient.class); @@ -58,7 +55,6 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); @@ -82,7 +78,6 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java index 988f1f7fec..182ac6a1c2 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.TaskExecuteThreadsFullPolicy; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -68,7 +67,7 @@ class WorkerTaskExecutorThreadPoolTest { protected MockWorkerTaskExecutor(Runnable runnable) { super(TaskExecutionContext.builder().taskInstanceId((int) System.nanoTime()).build(), new WorkerConfig(), - new WorkerMessageSender(), new TaskPluginManager(), new StorageOperate() { + new WorkerMessageSender(), new StorageOperate() { @Override public void createTenantDirIfNotExists(String tenantCode) { diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java index 592340214f..f761dd61d6 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java @@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHost import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; @@ -70,8 +69,6 @@ public class TaskInstanceOperationFunctionTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); - private WorkerTaskExecutorThreadPool workerManager = Mockito.mock(WorkerTaskExecutorThreadPool.class); private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); @@ -94,7 +91,6 @@ public class TaskInstanceOperationFunctionTest { WorkerTaskExecutorFactoryBuilder workerTaskExecutorFactoryBuilder = new WorkerTaskExecutorFactoryBuilder( workerConfig, workerMessageSender, - taskPluginManager, workerManager, storageOperate, workerRegistryClient); @@ -189,7 +185,6 @@ public class TaskInstanceOperationFunctionTest { WorkerTaskExecutorFactoryBuilder workerTaskExecutorFactoryBuilder = new WorkerTaskExecutorFactoryBuilder( workerConfig, workerMessageSender, - taskPluginManager, workerManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/test/resources/logback.xml b/dolphinscheduler-worker/src/test/resources/logback.xml index d63ea4a0b5..10fc24d974 100644 --- a/dolphinscheduler-worker/src/test/resources/logback.xml +++ b/dolphinscheduler-worker/src/test/resources/logback.xml @@ -22,7 +22,8 @@ - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - + [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n UTF-8 @@ -60,18 +61,15 @@ - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - + [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n UTF-8 - - - - - - + + From e66441a2c9ec38387b34f3712055af7308876efc Mon Sep 17 00:00:00 2001 From: privking <43061765+privking@users.noreply.github.com> Date: Sat, 20 Apr 2024 07:32:18 +0800 Subject: [PATCH 25/41] [FIX] Fix cannot recover a stopped workflow instance (#15880) --- .../workflow/instance/pause/recover/RecoverExecuteFunction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java index 149e1abd29..34bd2561b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java @@ -43,7 +43,7 @@ public class RecoverExecuteFunction implements ExecuteFunction Date: Mon, 22 Apr 2024 15:05:59 +0800 Subject: [PATCH 26/41] [Improvement][Spark] Support Local Spark Cluster (#15589) * [Improvement][Spark] Support Local Spark Cluster * remote default local from deploy mode --------- Co-authored-by: Rick Cheng --- docs/docs/en/guide/task/spark.md | 1 + docs/docs/zh/guide/task/spark.md | 1 + .../plugin/task/spark/SparkParameters.java | 5 + .../plugin/task/spark/SparkTask.java | 23 ++-- .../task/spark/SparkParametersTest.java | 1 - .../plugin/task/spark/SparkTaskTest.java | 109 +++++++++++++++--- .../src/locales/en_US/project.ts | 2 + .../src/locales/zh_CN/project.ts | 2 + .../task/components/node/fields/use-spark.ts | 26 +++++ .../task/components/node/format-data.ts | 1 + .../task/components/node/tasks/use-spark.ts | 3 +- 11 files changed, 151 insertions(+), 23 deletions(-) diff --git a/docs/docs/en/guide/task/spark.md b/docs/docs/en/guide/task/spark.md index 3e0f83b253..930f2cd0b0 100644 --- a/docs/docs/en/guide/task/spark.md +++ b/docs/docs/en/guide/task/spark.md @@ -24,6 +24,7 @@ Spark task type for executing Spark application. When executing the Spark task, |----------------------------|------------------------------------------------------------------------------------------------------------------------------------| | Program type | Supports Java, Scala, Python, and SQL. | | The class of main function | The **full path** of Main Class, the entry point of the Spark program. | +| Master | The The master URL for the cluster. | | Main jar package | The Spark jar package (upload by Resource Center). | | SQL scripts | SQL statements in .sql files that Spark sql runs. | | Deployment mode |

  • spark submit supports three modes: cluster, client and local.
  • spark sql supports client and local modes.
| diff --git a/docs/docs/zh/guide/task/spark.md b/docs/docs/zh/guide/task/spark.md index a392f55826..2f7b2ee346 100644 --- a/docs/docs/zh/guide/task/spark.md +++ b/docs/docs/zh/guide/task/spark.md @@ -23,6 +23,7 @@ Spark 任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支 - 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。 - 主函数的 Class：Spark 程序的入口 Main class 的全路径。 - 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。 +- Master：执行 Spark 集群的 Master Url。 - SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。 - 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。 (2) spark sql 支持 client 和 local 两种模式。 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java index c5fcb5b76b..873ba22c71 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java @@ -38,6 +38,11 @@ public class SparkParameters extends AbstractParameters { */ private String mainClass; + /** + * master url + */ + private String master; + /** * deploy mode local / cluster / client */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index a0d1f3fc77..3c5fc17698 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -124,22 +124,31 @@ public class SparkTask extends AbstractYarnTask { */ private List populateSparkOptions() { List args = new ArrayList<>(); - args.add(SparkConstants.MASTER); + // see https://spark.apache.org/docs/latest/submitting-applications.html + // TODO remove the option 'local' from deploy-mode String deployMode = StringUtils.isNotEmpty(sparkParameters.getDeployMode()) ? sparkParameters.getDeployMode() : SparkConstants.DEPLOY_MODE_LOCAL; + boolean onLocal = SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode); boolean onNativeKubernetes = StringUtils.isNotEmpty(sparkParameters.getNamespace()); - String masterUrl = onNativeKubernetes ? SPARK_ON_K8S_MASTER_PREFIX + - Config.fromKubeconfig(taskExecutionContext.getK8sTaskExecutionContext().getConfigYaml()).getMasterUrl() - : SparkConstants.SPARK_ON_YARN; + String masterUrl = StringUtils.isNotEmpty(sparkParameters.getMaster()) ? sparkParameters.getMaster() + : onLocal ? deployMode + : onNativeKubernetes + ? SPARK_ON_K8S_MASTER_PREFIX + Config + .fromKubeconfig( + taskExecutionContext.getK8sTaskExecutionContext().getConfigYaml()) + .getMasterUrl() + : SparkConstants.SPARK_ON_YARN; - if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode)) { - args.add(masterUrl); + args.add(SparkConstants.MASTER); + args.add(masterUrl); + + if (!onLocal) { args.add(SparkConstants.DEPLOY_MODE); + args.add(deployMode); } - args.add(deployMode); ProgramType programType = sparkParameters.getProgramType(); String mainClass = sparkParameters.getMainClass(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java index ab164f2eb5..19ec707c62 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java @@ -54,6 +54,5 @@ public class SparkParametersTest { resourceFilesList = sparkParameters.getResourceFilesList(); Assertions.assertNotNull(resourceFilesList); Assertions.assertEquals(3, resourceFilesList.size()); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index 78d8968e59..5138562564 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -17,16 +17,27 @@ package org.apache.dolphinscheduler.plugin.task.spark; +import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.TYPE_FILE; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -41,20 +52,33 @@ public class SparkTaskTest { Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("5536"); - SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); - sparkTask.init(); - Assertions.assertEquals( - "${SPARK_HOME}/bin/spark-sql " + - "--master yarn " + - "--deploy-mode client " + - "--conf spark.driver.cores=1 " + - "--conf spark.driver.memory=512M " + - "--conf spark.executor.instances=2 " + - "--conf spark.executor.cores=2 " + - "--conf spark.executor.memory=1G " + - "--name sparksql " + - "-f /tmp/5536_node.sql", - sparkTask.getScript()); + ResourceContext resourceContext = Mockito.mock(ResourceContext.class); + Mockito.when(taskExecutionContext.getResourceContext()).thenReturn(resourceContext); + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInLocal("test"); + Mockito.when(resourceContext.getResourceItem(any())).thenReturn(resourceItem); + + try (MockedStatic fileUtilsMockedStatic = Mockito.mockStatic(FileUtils.class)) { + fileUtilsMockedStatic + .when(() -> FileUtils + .readFileToString(any(File.class), any(Charset.class))) + .thenReturn("test"); + + SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); + sparkTask.init(); + Assertions.assertEquals( + "${SPARK_HOME}/bin/spark-sql " + + "--master yarn " + + "--deploy-mode client " + + "--conf spark.driver.cores=1 " + + "--conf spark.driver.memory=512M " + + "--conf spark.executor.instances=2 " + + "--conf spark.executor.cores=2 " + + "--conf spark.executor.memory=1G " + + "--name sparksql " + + "-f /tmp/5536_node.sql", + sparkTask.getScript()); + } } @Test @@ -86,11 +110,41 @@ public class SparkTaskTest { sparkTask.getScript()); } + @Test + public void testBuildCommandWithSparkSubmitMaster() { + String parameters = buildSparkParametersWithMaster(); + TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInStorage("/lib/dolphinscheduler-task-spark.jar"); + resourceItem.setResourceAbsolutePathInLocal("/lib/dolphinscheduler-task-spark.jar"); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem); + + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters); + Mockito.when(taskExecutionContext.getResourceContext()).thenReturn(resourceContext); + SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); + sparkTask.init(); + Assertions.assertEquals( + "${SPARK_HOME}/bin/spark-submit " + + "--master spark://localhost:7077 " + + "--deploy-mode client " + + "--class org.apache.dolphinscheduler.plugin.task.spark.SparkTaskTest " + + "--conf spark.driver.cores=1 " + + "--conf spark.driver.memory=512M " + + "--conf spark.executor.instances=2 " + + "--conf spark.executor.cores=2 " + + "--conf spark.executor.memory=1G " + + "--name spark " + + "/lib/dolphinscheduler-task-spark.jar", + sparkTask.getScript()); + } + private String buildSparkParametersWithSparkSql() { SparkParameters sparkParameters = new SparkParameters(); sparkParameters.setLocalParams(Collections.emptyList()); sparkParameters.setRawScript("selcet 11111;"); sparkParameters.setProgramType(ProgramType.SQL); + sparkParameters.setSqlExecutionType(TYPE_FILE); sparkParameters.setMainClass(""); sparkParameters.setDeployMode("client"); sparkParameters.setAppName("sparksql"); @@ -100,6 +154,13 @@ public class SparkTaskTest { sparkParameters.setNumExecutors(2); sparkParameters.setExecutorMemory("1G"); sparkParameters.setExecutorCores(2); + + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setResourceName("testSparkParameters1.jar"); + List resourceInfos = new ArrayList<>(Arrays.asList( + resourceInfo1)); + sparkParameters.setResourceList(resourceInfos); + return JSONUtils.toJsonString(sparkParameters); } @@ -122,4 +183,24 @@ public class SparkTaskTest { return JSONUtils.toJsonString(sparkParameters); } + private String buildSparkParametersWithMaster() { + SparkParameters sparkParameters = new SparkParameters(); + sparkParameters.setLocalParams(Collections.emptyList()); + sparkParameters.setProgramType(ProgramType.SCALA); + sparkParameters.setMainClass("org.apache.dolphinscheduler.plugin.task.spark.SparkTaskTest"); + sparkParameters.setDeployMode("client"); + sparkParameters.setAppName("spark"); + sparkParameters.setMaster("spark://localhost:7077"); + sparkParameters.setOthers(""); + sparkParameters.setDriverCores(1); + sparkParameters.setDriverMemory("512M"); + sparkParameters.setNumExecutors(2); + sparkParameters.setExecutorMemory("1G"); + sparkParameters.setExecutorCores(2); + ResourceInfo resourceInfo = new ResourceInfo(); + resourceInfo.setResourceName("/lib/dolphinscheduler-task-spark.jar"); + sparkParameters.setMainJar(resourceInfo); + return JSONUtils.toJsonString(sparkParameters); + } + } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index cb50b19fc7..7a39752526 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -447,6 +447,8 @@ export default { timeout_period_tips: 'Timeout must be a positive integer', script: 'Script', script_tips: 'Please enter script(required)', + master: 'Master', + master_tips: 'Please enter master url(required)', init_script: 'Initialization script', init_script_tips: 'Please enter initialization script', resources: 'Resources', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 6865a49abc..50e0a821ef 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -437,6 +437,8 @@ export default { timeout_period_tips: '超时时长必须为正整数', script: '脚本', script_tips: '请输入脚本（必填）', + master: 'Master', + master_tips: '请输入master url（必填）', init_script: '初始化脚本', init_script_tips: '请输入初始化脚本', resources: '资源', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts index ad7fb77fa9..ab89e69e6d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts @@ -37,6 +37,10 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { model.programType === 'PYTHON' || model.programType === 'SQL' ? 0 : 24 ) + const masterSpan = computed(() => + model.programType === 'PYTHON' || model.programType === 'SQL' ? 0 : 24 + ) + const mainArgsSpan = computed(() => (model.programType === 'SQL' ? 0 : 24)) const rawScriptSpan = computed(() => @@ -138,6 +142,28 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { message: t('project.node.script_tips') } }, + { + type: 'input', + field: 'master', + span: masterSpan, + name: t('project.node.master'), + props: { + placeholder: t('project.node.master_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: false, + validator(validate: any, value: string) { + if ( + model.programType !== 'PYTHON' && + !value && + model.programType !== 'SQL' + ) { + return new Error(t('project.node.master_tips')) + } + } + } + }, useDeployMode(24, ref(true), showCluster), useNamespace(), { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index ef9e5dec61..2ab1712b6f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -68,6 +68,7 @@ export function formatParams(data: INodeData): { } if (data.taskType === 'SPARK') { + taskParams.master = data.master taskParams.driverCores = data.driverCores taskParams.driverMemory = data.driverMemory taskParams.numExecutors = data.numExecutors diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts index 05aa0fe1c6..1e5c929b0f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts @@ -45,7 +45,8 @@ export function useSpark({ timeout: 30, programType: 'SCALA', rawScript: '', - deployMode: 'local', + master: '', + deployMode: '', driverCores: 1, driverMemory: '512M', numExecutors: 2, From e2c8b080f90c71aa6045609a84ffde9272892fea Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 22 Apr 2024 17:05:24 +0800 Subject: [PATCH 27/41] [DSIP-31] Reduce the thread pool size of hikari (#15890) --- .../src/main/resources/application.yaml | 8 -------- dolphinscheduler-api/src/main/resources/application.yaml | 8 -------- .../src/main/resources/application.yaml | 8 -------- .../src/main/resources/application.yaml | 8 -------- .../src/test/resources/application-mysql.yaml | 8 -------- .../src/test/resources/application-postgresql.yaml | 8 -------- 6 files changed, 48 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 0b28d6c8b0..0dbb6988ce 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -30,15 +30,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 # Mybatis-plus configuration, you don't need to change it mybatis-plus: diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 61f9c8a592..e38d0c5a8e 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -51,15 +51,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 quartz: auto-startup: false job-store-type: jdbc diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index c2d8f5e787..f18c6ef61d 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -29,15 +29,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 quartz: job-store-type: jdbc jdbc: diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml index 136a4c5fd4..23680728dc 100644 --- a/dolphinscheduler-tools/src/main/resources/application.yaml +++ b/dolphinscheduler-tools/src/main/resources/application.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 # Mybatis-plus configuration, you don't need to change it mybatis-plus: diff --git a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml index 0b553c2bda..d33a68a7f8 100644 --- a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 mybatis-plus: mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml diff --git a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml index 21daaa0606..a8be29014f 100644 --- a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 mybatis-plus: mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml From e9d85914d78197314bee585500d94a6a90733789 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 22 Apr 2024 17:41:17 +0800 Subject: [PATCH 28/41] Fix queryByTypeAndJobId might error due to multiple result (#15883) --- .../dao/mapper/TriggerRelationMapper.java | 2 +- .../dao/mapper/TriggerRelationMapperTest.java | 26 ++++----- dolphinscheduler-dist/release-docs/LICENSE | 2 +- .../process/TriggerRelationService.java | 4 +- .../process/TriggerRelationServiceImpl.java | 41 ++++++++++---- .../process/TriggerRelationServiceTest.java | 56 ++++++++----------- pom.xml | 7 +++ tools/dependencies/known-dependencies.txt | 4 +- 8 files changed, 79 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java index 10a0acf47f..912ef28101 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java @@ -36,7 +36,7 @@ public interface TriggerRelationMapper extends BaseMapper { * @param jobId * @return */ - TriggerRelation queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); + List queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); /** * query triggerRelation by code diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java index d3f4fcc666..7e31b64a23 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; -import java.util.List; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -67,9 +67,9 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTypeAndJobId() { TriggerRelation expectRelation = createTriggerRelation(); - TriggerRelation actualRelation = triggerRelationMapper.queryByTypeAndJobId( - expectRelation.getTriggerType(), expectRelation.getJobId()); - Assertions.assertEquals(expectRelation, actualRelation); + assertThat( + triggerRelationMapper.queryByTypeAndJobId(expectRelation.getTriggerType(), expectRelation.getJobId())) + .containsExactly(expectRelation); } /** @@ -80,9 +80,8 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTriggerRelationCode() { TriggerRelation expectRelation = createTriggerRelation(); - List actualRelations = triggerRelationMapper.queryByTriggerRelationCode( - expectRelation.getTriggerCode()); - Assertions.assertEquals(actualRelations.size(), 1); + assertThat(triggerRelationMapper.queryByTriggerRelationCode(expectRelation.getTriggerCode())) + .containsExactly(expectRelation); } /** @@ -93,17 +92,15 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTriggerRelationCodeAndType() { TriggerRelation expectRelation = createTriggerRelation(); - List actualRelations = triggerRelationMapper.queryByTriggerRelationCodeAndType( - expectRelation.getTriggerCode(), expectRelation.getTriggerType()); - Assertions.assertEquals(actualRelations.size(), 1); + assertThat(triggerRelationMapper.queryByTriggerRelationCodeAndType(expectRelation.getTriggerCode(), + expectRelation.getTriggerType())).containsExactly(expectRelation); } @Test public void testUpsert() { TriggerRelation expectRelation = createTriggerRelation(); triggerRelationMapper.upsert(expectRelation); - TriggerRelation actualRelation = triggerRelationMapper.selectById(expectRelation.getId()); - Assertions.assertEquals(expectRelation, actualRelation); + assertThat(triggerRelationMapper.selectById(expectRelation.getId())).isEqualTo(expectRelation); } /** @@ -113,8 +110,7 @@ public class TriggerRelationMapperTest extends BaseDaoTest { public void testDelete() { TriggerRelation expectRelation = createTriggerRelation(); triggerRelationMapper.deleteById(expectRelation.getId()); - TriggerRelation actualRelation = triggerRelationMapper.selectById(expectRelation.getId()); - Assertions.assertNull(actualRelation); + assertThat(triggerRelationMapper.selectById(expectRelation.getId())).isNull(); } /** diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 8856138aa4..84832b2b32 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -528,7 +528,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. nimbus-jose-jwt 9.22: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.22, Apache 2.0 woodstox-core 6.4.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/6.4.0, Apache 2.0 auto-value 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value/1.10.1, Apache 2.0 - auto-value-annotations 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.1, Apache 2.0 + auto-value-annotations 1.10.4: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.4, Apache 2.0 conscrypt-openjdk-uber 2.5.2: https://mvnrepository.com/artifact/org.conscrypt/conscrypt-openjdk-uber/2.5.2, Apache 2.0 gapic-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/gapic-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0 google-api-client 2.2.0: https://mvnrepository.com/artifact/com.google.api-client/google-api-client/2.2.0, Apache 2.0 diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java index b78f477931..6de2506afd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.service.process; import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; +import java.util.List; + import org.springframework.stereotype.Component; /** @@ -30,7 +32,7 @@ public interface TriggerRelationService { void saveTriggerToDb(ApiTriggerType type, Long triggerCode, Integer jobId); - TriggerRelation queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId); + List queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId); int saveCommandTrigger(Integer commandId, Integer processInstanceId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java index df41c41eef..0344ea87ea 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java @@ -21,14 +21,20 @@ import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; import org.apache.dolphinscheduler.dao.mapper.TriggerRelationMapper; +import org.apache.commons.collections4.CollectionUtils; + import java.util.Date; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** - * Trigger relation operator to db + * Trigger relation operator to db */ +@Slf4j @Component public class TriggerRelationServiceImpl implements TriggerRelationService { @@ -45,29 +51,44 @@ public class TriggerRelationServiceImpl implements TriggerRelationService { triggerRelation.setUpdateTime(new Date()); triggerRelationMapper.upsert(triggerRelation); } + @Override - public TriggerRelation queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId) { + public List queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId) { return triggerRelationMapper.queryByTypeAndJobId(apiTriggerType.getCode(), jobId); } @Override public int saveCommandTrigger(Integer commandId, Integer processInstanceId) { - TriggerRelation exist = queryByTypeAndJobId(ApiTriggerType.PROCESS, processInstanceId); - if (exist == null) { + List existTriggers = queryByTypeAndJobId(ApiTriggerType.PROCESS, processInstanceId); + if (CollectionUtils.isEmpty(existTriggers)) { return 0; } - saveTriggerToDb(ApiTriggerType.COMMAND, exist.getTriggerCode(), commandId); - return 1; + existTriggers.forEach(triggerRelation -> saveTriggerToDb(ApiTriggerType.COMMAND, + triggerRelation.getTriggerCode(), commandId)); + int triggerRelationSize = existTriggers.size(); + if (triggerRelationSize > 1) { + // Fix https://github.com/apache/dolphinscheduler/issues/15864 + // This case shouldn't happen + log.error("The PROCESS TriggerRelation of command: {} is more than one", commandId); + } + return existTriggers.size(); } @Override public int saveProcessInstanceTrigger(Integer commandId, Integer processInstanceId) { - TriggerRelation exist = queryByTypeAndJobId(ApiTriggerType.COMMAND, commandId); - if (exist == null) { + List existTriggers = queryByTypeAndJobId(ApiTriggerType.COMMAND, commandId); + if (CollectionUtils.isEmpty(existTriggers)) { return 0; } - saveTriggerToDb(ApiTriggerType.PROCESS, exist.getTriggerCode(), processInstanceId); - return 1; + existTriggers.forEach(triggerRelation -> saveTriggerToDb(ApiTriggerType.PROCESS, + triggerRelation.getTriggerCode(), processInstanceId)); + int triggerRelationSize = existTriggers.size(); + if (triggerRelationSize > 1) { + // Fix https://github.com/apache/dolphinscheduler/issues/15864 + // This case shouldn't happen + log.error("The COMMAND TriggerRelation of command: {} is more than one", commandId); + } + return existTriggers.size(); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java index 8f4790111c..4f3bbb0bc7 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java @@ -17,24 +17,26 @@ package org.apache.dolphinscheduler.service.process; +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; import org.apache.dolphinscheduler.dao.mapper.TriggerRelationMapper; -import org.apache.dolphinscheduler.service.cron.CronUtilsTest; import java.util.Date; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * Trigger Relation Service Test @@ -43,8 +45,6 @@ import org.slf4j.LoggerFactory; @MockitoSettings(strictness = Strictness.LENIENT) public class TriggerRelationServiceTest { - private static final Logger logger = LoggerFactory.getLogger(CronUtilsTest.class); - @InjectMocks private TriggerRelationServiceImpl triggerRelationService; @Mock @@ -52,47 +52,37 @@ public class TriggerRelationServiceTest { @Test public void saveTriggerToDb() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); + doNothing().when(triggerRelationMapper).upsert(any()); triggerRelationService.saveTriggerToDb(ApiTriggerType.COMMAND, 1234567890L, 100); } @Test public void queryByTypeAndJobId() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); - TriggerRelation triggerRelation1 = triggerRelationService.queryByTypeAndJobId( - ApiTriggerType.PROCESS, 100); - Assertions.assertNotNull(triggerRelation1); - TriggerRelation triggerRelation2 = triggerRelationService.queryByTypeAndJobId( - ApiTriggerType.PROCESS, 200); - Assertions.assertNull(triggerRelation2); + assertThat(triggerRelationService.queryByTypeAndJobId(ApiTriggerType.PROCESS, 100)).hasSize(1); + assertThat(triggerRelationService.queryByTypeAndJobId(ApiTriggerType.PROCESS, 200)).isEmpty(); } @Test public void saveCommandTrigger() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); - int result = -1; - result = triggerRelationService.saveCommandTrigger(1234567890, 100); - Assertions.assertTrue(result > 0); - result = triggerRelationService.saveCommandTrigger(1234567890, 200); - Assertions.assertTrue(result == 0); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); + assertThat(triggerRelationService.saveCommandTrigger(1234567890, 100)).isAtLeast(1); + assertThat(triggerRelationService.saveCommandTrigger(1234567890, 200)).isEqualTo(0); } @Test public void saveProcessInstanceTrigger() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.COMMAND.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); - int result = -1; - result = triggerRelationService.saveProcessInstanceTrigger(100, 1234567890); - Assertions.assertTrue(result > 0); - result = triggerRelationService.saveProcessInstanceTrigger(200, 1234567890); - Assertions.assertTrue(result == 0); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.COMMAND.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); + assertThat(triggerRelationService.saveProcessInstanceTrigger(100, 1234567890)).isAtLeast(1); + assertThat(triggerRelationService.saveProcessInstanceTrigger(200, 1234567890)).isEqualTo(0); } private TriggerRelation getTriggerTdoDb() { diff --git a/pom.xml b/pom.xml index 4a71741f8e..dd4cb1adf1 100755 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 7.1.2 1.18.20 4.2.0 + 1.4.2 apache ${project.name} ${project.version} @@ -372,6 +373,12 @@ ${awaitility.version} test + + com.google.truth + truth + ${truth.version} + test + diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 6c2e89fb37..33987119db 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -9,7 +9,7 @@ animal-sniffer-annotations-1.19.jar annotations-2.17.282.jar annotations-13.0.jar apache-client-2.17.282.jar -asm-9.1.jar +asm-9.6.jar aspectjweaver-1.9.7.jar aspectjrt-1.9.7.jar auth-2.17.282.jar @@ -434,7 +434,7 @@ woodstox-core-6.4.0.jar azure-core-management-1.10.1.jar api-common-2.6.0.jar auto-value-1.10.1.jar -auto-value-annotations-1.10.1.jar +auto-value-annotations-1.10.4.jar bcpkix-jdk15on-1.67.jar bcprov-jdk15on-1.67.jar conscrypt-openjdk-uber-2.5.2.jar From 59f060e278ad0c2400e4dea6359ffda8b1f71d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Tue, 23 Apr 2024 15:10:08 +0800 Subject: [PATCH 29/41] [Improvement] Fix alert code smell --- .../plugin/alert/dingtalk/DingTalkSender.java | 28 ++++++------------- .../alert/dingtalk/DingTalkSenderTest.java | 3 +- .../plugin/alert/email/EmailConstants.java | 2 -- .../plugin/alert/email/MailSender.java | 6 ++-- .../plugin/alert/email/ExcelUtilsTest.java | 11 ++++++-- .../plugin/alert/email/MailUtilsTest.java | 22 +++++++-------- .../plugin/alert/feishu/FeiShuSender.java | 3 +- .../plugin/alert/http/HttpSender.java | 9 +++--- .../prometheus/PrometheusAlertSender.java | 3 +- .../plugin/alert/script/ProcessUtilsTest.java | 3 +- .../plugin/alert/slack/SlackSender.java | 5 ++-- .../plugin/alert/telegram/TelegramSender.java | 2 +- .../alert/wechat/WeChatAlertConstants.java | 2 -- .../plugin/alert/wechat/WeChatSender.java | 27 ++++++------------ .../common/constants/Constants.java | 5 ---- .../common/utils/FileUtils.java | 3 +- .../common/utils/HttpUtils.java | 3 +- 17 files changed, 60 insertions(+), 77 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index c0070ac11c..c8ded8cfad 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -48,6 +48,8 @@ import java.util.Objects; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -189,7 +191,7 @@ public final class DingTalkSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -317,15 +319,17 @@ public final class DingTalkSender { String sign = org.apache.commons.lang3.StringUtils.EMPTY; try { Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); - byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); - sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + byte[] signData = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); + sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error("generate sign error, message:{}", e); } return url + "×tamp=" + timestamp + "&sign=" + sign; } + @Getter + @Setter static final class DingTalkSendMsgResponse { private Integer errcode; @@ -334,22 +338,6 @@ public final class DingTalkSender { public DingTalkSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - @Override public boolean equals(final Object o) { if (o == this) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index 7a8df0549c..cd30105c7a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; import org.apache.dolphinscheduler.alert.api.AlertResult; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -47,7 +48,7 @@ public class DingTalkSenderTest { @Test public void testSend() { DingTalkSender dingTalkSender = new DingTalkSender(dingTalkConfig); - dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8"); + dingTalkSender.sendDingTalkMsg("keyWord+Welcome", StandardCharsets.UTF_8.name()); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkSender = new DingTalkSender(dingTalkConfig); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test"); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java index 94eb4efa39..3eec7022fd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java @@ -56,8 +56,6 @@ public final class EmailConstants { public static final String TABLE_BODY_HTML_TAIL = ""; - public static final String UTF_8 = "UTF-8"; - public static final String EXCEL_SUFFIX_XLSX = ".xlsx"; public static final String SINGLE_SLASH = "/"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index 2e400efbce..8826a44fba 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -34,6 +34,7 @@ import org.apache.commons.mail.HtmlEmail; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -171,7 +172,7 @@ public final class MailSender { Session session = getSession(); email.setMailSession(session); email.setFrom(mailSenderEmail); - email.setCharset(EmailConstants.UTF_8); + email.setCharset(StandardCharsets.UTF_8.name()); if (CollectionUtils.isNotEmpty(receivers)) { // receivers mail for (String receiver : receivers) { @@ -344,7 +345,8 @@ public final class MailSender { ExcelUtils.genExcelFile(content, randomFilename, xlsFilePath); part2.attachFile(file); - part2.setFileName(MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, EmailConstants.UTF_8, "B")); + part2.setFileName( + MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, StandardCharsets.UTF_8.name(), "B")); // add components to collection partList.addBodyPart(part1); partList.addBodyPart(part2); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java index f428a16d92..f28df77de9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java @@ -66,8 +66,15 @@ public class ExcelUtilsTest { @Test public void testGenExcelFileByCheckDir() { - ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls"); - File file = new File("/tmp/xls" + EmailConstants.SINGLE_SLASH + "t" + EmailConstants.EXCEL_SUFFIX_XLSX); + String path = "/tmp/xls"; + ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", path); + File file = + new File( + path + + EmailConstants.SINGLE_SLASH + + "t" + + EmailConstants.EXCEL_SUFFIX_XLSX); file.delete(); + Assertions.assertFalse(file.exists()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index f562999be0..acc255ae0e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -18,10 +18,9 @@ package org.apache.dolphinscheduler.plugin.alert.email; import org.apache.dolphinscheduler.alert.api.AlertConstants; +import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.api.ShowType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate; -import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate; import java.util.ArrayList; import java.util.HashMap; @@ -42,7 +41,6 @@ public class MailUtilsTest { private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class); static MailSender mailSender; private static Map emailConfig = new HashMap<>(); - private static AlertTemplate alertTemplate; @BeforeAll public static void initEmailConfig() { @@ -59,7 +57,6 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, "347801120@qq.com"); emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, "347801120@qq.com"); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp()); - alertTemplate = new DefaultHTMLTemplate(); mailSender = new MailSender(emailConfig); } @@ -77,9 +74,10 @@ public class MailUtilsTest { + "\"Host: 192.168.xx.xx\"," + "\"Notify group :4\"]"; - mailSender.sendMails( + AlertResult alertResult = mailSender.sendMails( "Mysql Exception", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -108,7 +106,8 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "user"); emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd"); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } public String list2String() { @@ -134,7 +133,6 @@ public class MailUtilsTest { logger.info(mapjson); return mapjson; - } @Test @@ -143,7 +141,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -151,7 +150,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -159,7 +159,7 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } - } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index 14a1d63ff0..369060843c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -31,6 +31,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -161,7 +162,7 @@ public final class FeiShuSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java index 999a0c9599..a1de852407 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -58,7 +59,6 @@ public final class HttpSender { * request type get */ private static final String REQUEST_TYPE_GET = "GET"; - private static final String DEFAULT_CHARSET = "utf-8"; private final String headerParams; private final String bodyParams; private final String contentField; @@ -124,7 +124,7 @@ public final class HttpSender { CloseableHttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, DEFAULT_CHARSET); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } private void createHttpRequest(String msg) throws MalformedURLException, URISyntaxException { @@ -157,7 +157,8 @@ public final class HttpSender { type = URL_SPLICE_CHAR; } try { - url = String.format("%s%s%s=%s", url, type, contentField, URLEncoder.encode(msg, DEFAULT_CHARSET)); + url = String.format("%s%s%s=%s", url, type, contentField, + URLEncoder.encode(msg, StandardCharsets.UTF_8.name())); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -190,7 +191,7 @@ public final class HttpSender { } // set msg content field objectNode.put(contentField, msg); - StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), DEFAULT_CHARSET); + StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), StandardCharsets.UTF_8); ((HttpPost) httpRequest).setEntity(entity); } catch (Exception e) { log.error("send http alert msg exception : {}", e.getMessage()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java index d27745049e..1106e6799f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java @@ -34,6 +34,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -90,7 +91,7 @@ public class PrometheusAlertSender { } HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); log.error( "Prometheus alert manager send alert failed, http status code: {}, title: {} ,content: {}, resp: {}", diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java index a34f062264..3d85d5d638 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java @@ -33,6 +33,7 @@ public class ProcessUtilsTest { @Test public void testExecuteScript() { - ProcessUtils.executeScript(cmd); + int code = ProcessUtils.executeScript(cmd); + assert code != -1; } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java index 60b2f8281a..4096ccc998 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java @@ -29,6 +29,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -81,11 +82,11 @@ public final class SlackSender { } HttpPost httpPost = new HttpPost(webHookUrl); - httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), "UTF-8")); + httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, "UTF-8"); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } catch (Exception e) { log.error("Send message to slack error.", e); return "System Exception"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 8aba9f5c2b..129bc62c1c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -153,7 +153,7 @@ public final class TelegramSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java index 7f5eaef4f9..76ad480015 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java @@ -23,8 +23,6 @@ public final class WeChatAlertConstants { static final String MARKDOWN_ENTER = "\n"; - static final String CHARSET = "UTF-8"; - static final String WE_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}"; static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index 4b49e0436d..c5ffec1f46 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -38,6 +38,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -45,6 +46,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -85,12 +88,12 @@ public final class WeChatSender { CloseableHttpClient httpClient = HttpClients.custom().setRetryHandler(HttpServiceRetryStrategy.retryStrategy).build()) { HttpPost httpPost = new HttpPost(url); - httpPost.setEntity(new StringEntity(data, WeChatAlertConstants.CHARSET)); + httpPost.setEntity(new StringEntity(data, StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -142,7 +145,7 @@ public final class WeChatSender { HttpGet httpGet = new HttpGet(url); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } @@ -259,6 +262,8 @@ public final class WeChatSender { return null; } + @Getter + @Setter static final class WeChatSendMsgResponse { private Integer errcode; @@ -267,22 +272,6 @@ public final class WeChatSender { public WeChatSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - public boolean equals(final Object o) { if (o == this) { return true; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index b5bbf740e9..ebf668a312 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -242,11 +242,6 @@ public final class Constants { */ public static final String HTTP_X_REAL_IP = "X-Real-IP"; - /** - * UTF-8 - */ - public static final String UTF_8 = "UTF-8"; - /** * user name regex */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index fee1d9a95c..60629576d9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPA import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE; -import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS; import org.apache.commons.io.IOUtils; @@ -207,7 +206,7 @@ public class FileUtils { while ((length = inputStream.read(buffer)) != -1) { output.write(buffer, 0, length); } - return output.toString(UTF_8); + return output.toString(StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException(e); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 5c79fff951..e5d2256150 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -36,6 +36,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -143,7 +144,7 @@ public class HttpUtils { } HttpEntity entity = response.getEntity(); - return entity != null ? EntityUtils.toString(entity, Constants.UTF_8) : null; + return entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null; } catch (IOException e) { log.error("Error executing HTTP GET request", e); return null; From 1d13ef09c3607870f7309d9f41dc5deac8d209ee Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Wed, 24 Apr 2024 23:31:56 +0800 Subject: [PATCH 30/41] [TEST] increase coverage of project parameter service test (#15905) Co-authored-by: abzymeinsjtu --- .../impl/ProjectParameterServiceImpl.java | 6 +- .../service/ProjectParameterServiceTest.java | 204 +++++++++++++----- 2 files changed, 157 insertions(+), 53 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e0011096e4..17d1d04712 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -227,11 +227,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj } for (ProjectParameter projectParameter : projectParameterList) { - try { - this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); - } catch (Exception e) { - throw new ServiceException(Status.DELETE_PROJECT_PARAMETER_ERROR, e.getMessage()); - } + this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index 7a3fb1b68d..ca51690ce6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -17,27 +17,40 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl; import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectParameter; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; -import org.junit.jupiter.api.Assertions; +import java.util.Collections; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class ProjectParameterServiceTest { @@ -60,92 +73,187 @@ public class ProjectParameterServiceTest { public void testCreateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); + // PERMISSION DENIED + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + + // CODE GENERATION ERROR + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); + } + + // PROJECT_PARAMETER_ALREADY_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // INSERT DATA ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testUpdateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // PROJECT_UPDATE_ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); + assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); ProjectParameter projectParameter = (ProjectParameter) result.getData(); - Assertions.assertNotNull(projectParameter.getOperator()); - Assertions.assertNotNull(projectParameter.getUpdateTime()); + assertNotNull(projectParameter.getOperator()); + assertNotNull(projectParameter.getUpdateTime()); } @Test public void testDeleteProjectParametersByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + + // DATABASE OPERATION ERROR + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(-1); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.DELETE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testQueryProjectParameterByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), - Mockito.any())).thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + Result result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())).thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + @Test + public void testQueryProjectParameterListPaging() { + User loginUser = getGeneralUser(); + Integer pageSize = 10; + Integer pageNo = 1; + + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + + Result result = + projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // SUCCESS + when(projectService.hasProjectAndPerm(any(), any(), any(Result.class), any())) + .thenReturn(true); + + Page page = new Page<>(pageNo, pageSize); + page.setRecords(Collections.singletonList(getProjectParameter())); + + when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page); + result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, + null); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); + } + + @Test + public void testBatchDeleteProjectParametersByCodes() { + User loginUser = getGeneralUser(); + + Result result = projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, ""); + assertEquals(Status.PROJECT_PARAMETER_CODE_EMPTY.getCode(), result.getCode()); + + when(projectParameterMapper.queryByCodes(any())).thenReturn(Collections.singletonList(getProjectParameter())); + + AssertionsHelper.assertThrowsServiceException(Status.PROJECT_PARAMETER_NOT_EXISTS, + () -> projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1,2")); + + projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1"); } private Project getProject(long projectCode) { From b3b8c0784dfc31b516b13601a311bc78550bf931 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 11:05:51 +0800 Subject: [PATCH 31/41] Fix kill dynamic task doesn't kill the wait to run workflow instances (#15896) --- .../dao/mapper/CommandMapper.java | 9 +++- .../dao/mapper/CommandMapper.xml | 7 +++ .../dao/mapper/CommandMapperTest.java | 11 +++- .../runner/task/dynamic/DynamicLogicTask.java | 51 ++++++++++++++++++- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index a8490cbef7..9fb6643227 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -34,8 +34,9 @@ public interface CommandMapper extends BaseMapper { /** * count command state - * @param startTime startTime - * @param endTime endTime + * + * @param startTime startTime + * @param endTime endTime * @param projectCodes projectCodes * @return CommandCount list */ @@ -46,15 +47,19 @@ public interface CommandMapper extends BaseMapper { /** * query command page + * * @return */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); /** * query command page by slot + * * @return command list */ List queryCommandPageBySlot(@Param("limit") int limit, @Param("masterCount") int masterCount, @Param("thisMasterSlot") int thisMasterSlot); + + void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index c950f66413..56db890ef0 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -47,4 +47,11 @@ order by process_instance_priority, id asc limit #{limit} + + delete from t_ds_command + where process_instance_id in + + #{i} + + diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 3d45477d85..2d367e46e4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.dao.mapper; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; @@ -173,6 +175,14 @@ public class CommandMapperTest extends BaseDaoTest { toTestQueryCommandPageBySlot(masterCount, thisMasterSlot); } + @Test + void deleteByWorkflowInstanceIds() { + Command command = createCommand(); + assertThat(commandMapper.selectList(null)).isNotEmpty(); + commandMapper.deleteByWorkflowInstanceIds(Lists.newArrayList(command.getProcessInstanceId())); + assertThat(commandMapper.selectList(null)).isEmpty(); + } + private boolean toTestQueryCommandPageBySlot(int masterCount, int thisMasterSlot) { Command command = createCommand(); Integer id = command.getId(); @@ -280,5 +290,4 @@ public class CommandMapperTest extends BaseDaoTest { return command; } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java index 3baa10b343..12cae5c53e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java @@ -252,12 +252,61 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { @Override public void kill() { try { - changeRunningSubprocessInstancesToStop(WorkflowExecutionStatus.READY_STOP); + doKillSubWorkflowInstances(); } catch (MasterTaskExecuteException e) { log.error("kill {} error", taskInstance.getName(), e); } } + private void doKillSubWorkflowInstances() throws MasterTaskExecuteException { + List existsSubProcessInstanceList = + subWorkflowService.getAllDynamicSubWorkflow(processInstance.getId(), taskInstance.getTaskCode()); + if (CollectionUtils.isEmpty(existsSubProcessInstanceList)) { + return; + } + + commandMapper.deleteByWorkflowInstanceIds( + existsSubProcessInstanceList.stream().map(ProcessInstance::getId).collect(Collectors.toList())); + + List runningSubProcessInstanceList = + subWorkflowService.filterRunningProcessInstances(existsSubProcessInstanceList); + doKillRunningSubWorkflowInstances(runningSubProcessInstanceList); + + List waitToRunProcessInstances = + subWorkflowService.filterWaitToRunProcessInstances(existsSubProcessInstanceList); + doKillWaitToRunSubWorkflowInstances(waitToRunProcessInstances); + + this.haveBeenCanceled = true; + } + + private void doKillRunningSubWorkflowInstances(List runningSubProcessInstanceList) throws MasterTaskExecuteException { + for (ProcessInstance subProcessInstance : runningSubProcessInstanceList) { + subProcessInstance.setState(WorkflowExecutionStatus.READY_STOP); + processInstanceDao.updateById(subProcessInstance); + if (subProcessInstance.getState().isFinished()) { + log.info("The process instance [{}] is finished, no need to stop", subProcessInstance.getId()); + continue; + } + try { + sendToSubProcess(taskExecutionContext, subProcessInstance); + log.info("Success send [{}] request to SubWorkflow's master: {}", WorkflowExecutionStatus.READY_STOP, + subProcessInstance.getHost()); + } catch (Exception e) { + throw new MasterTaskExecuteException( + String.format("Send stop request to SubWorkflow's master: %s failed", + subProcessInstance.getHost()), + e); + } + } + } + + private void doKillWaitToRunSubWorkflowInstances(List waitToRunWorkflowInstances) { + for (ProcessInstance subProcessInstance : waitToRunWorkflowInstances) { + subProcessInstance.setState(WorkflowExecutionStatus.STOP); + processInstanceDao.updateById(subProcessInstance); + } + } + private void changeRunningSubprocessInstancesToStop(WorkflowExecutionStatus stopStatus) throws MasterTaskExecuteException { this.haveBeenCanceled = true; List existsSubProcessInstanceList = From 446f6ba72b8347ad817d8f62cdd8d258ace7adf4 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 18:14:48 +0800 Subject: [PATCH 32/41] Fix auto create tennat concurrently will cause the task failed (#15909) --- .../common/utils/OSUtils.java | 25 ++++++++----------- .../utils/TaskExecutionContextUtils.java | 2 +- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index beca53c3fd..dbfcea2ed8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -183,11 +183,10 @@ public class OSUtils { * * @param userName user name */ - public static void createUserIfAbsent(String userName) { + public static synchronized void createUserIfAbsent(String userName) { // if not exists this user, then create if (!getUserList().contains(userName)) { - boolean isSuccess = createUser(userName); - log.info("create user {} {}", userName, isSuccess ? "success" : "fail"); + createUser(userName); } } @@ -197,13 +196,12 @@ public class OSUtils { * @param userName user name * @return true if creation was successful, otherwise false */ - public static boolean createUser(String userName) { + public static void createUser(String userName) { try { String userGroup = getGroup(); if (StringUtils.isEmpty(userGroup)) { - String errorLog = String.format("%s group does not exist for this operating system.", userGroup); - log.error(errorLog); - return false; + throw new UnsupportedOperationException( + "There is no userGroup exist cannot create tenant, please create userGroupFirst"); } if (SystemUtils.IS_OS_MAC) { createMacUser(userName, userGroup); @@ -212,18 +210,17 @@ public class OSUtils { } else { createLinuxUser(userName, userGroup); } - return true; + log.info("Create tenant {} under userGroup: {} success", userName, userGroup); } catch (Exception e) { - log.error(e.getMessage(), e); + throw new RuntimeException("Create tenant: {} failed", e); } - return false; } /** * create linux user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -237,7 +234,7 @@ public class OSUtils { /** * create mac user (Supports Mac OSX 10.10+) * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -256,7 +253,7 @@ public class OSUtils { /** * create windows user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -304,7 +301,7 @@ public class OSUtils { * get sudo command * * @param tenantCode tenantCode - * @param command command + * @param command command * @return result of sudo execute command */ public static String getSudoCmd(String tenantCode, String command) { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 3cda6ac099..f43b19c444 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -78,7 +78,7 @@ public class TaskExecutionContextUtils { throw ex; } catch (Exception ex) { throw new TaskException( - String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode())); + String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode()), ex); } } From d1135eabc7df9358fd210e423785d1067af64698 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 19:39:56 +0800 Subject: [PATCH 33/41] [DSIP-34] Change required_approving_review_count to 2 (#15914) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index cf409dc6a3..84619447be 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,4 +49,4 @@ github: - "Mergeable: milestone-label-check" required_pull_request_reviews: dismiss_stale_reviews: true - required_approving_review_count: 1 + required_approving_review_count: 2 From b29965bdce2b5b83c1ffe237265e6f53f01e11cf Mon Sep 17 00:00:00 2001 From: DaqianLiao <360989637@qq.com> Date: Fri, 26 Apr 2024 11:24:05 +0800 Subject: [PATCH 34/41] [Fix] In updateWorkerNodes method, the workerNodeInfoWriteLock should be used. #15898 (#15903) Co-authored-by: answerliao --- .../server/master/registry/ServerNodeManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java index 258acd8f6e..f066f0403d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java @@ -245,7 +245,7 @@ public class ServerNodeManager implements InitializingBean { } private void updateWorkerNodes() { - workerGroupWriteLock.lock(); + workerNodeInfoWriteLock.lock(); try { Map workerNodeMaps = registryClient.getServerMaps(RegistryNodeType.WORKER); for (Map.Entry entry : workerNodeMaps.entrySet()) { @@ -254,7 +254,7 @@ public class ServerNodeManager implements InitializingBean { workerNodeInfo.put(nodeAddress, workerHeartBeat); } } finally { - workerGroupWriteLock.unlock(); + workerNodeInfoWriteLock.unlock(); } } From 7a55adeae9d9c5e8bc813d952322c8da1aabdc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E8=80=90?= <46134044+sdhzwc@users.noreply.github.com> Date: Sun, 28 Apr 2024 11:02:44 +0800 Subject: [PATCH 35/41] [Improvement-15919][datasource] Improvement datasource get name (#15920) --- .../AbstractDataSourceProcessor.java | 2 +- .../api/plugin/DataSourceClientProvider.java | 8 +-- .../AthenaDataSourceChannelFactory.java | 3 +- .../AzureSQLDataSourceChannelFactory.java | 3 +- .../ClickHouseDataSourceChannelFactory.java | 3 +- .../DamengDataSourceChannelFactory.java | 2 +- .../DatabendDataSourceChannelFactory.java | 3 +- .../DatabendDataSourceProcessorTest.java | 2 +- .../db2/DB2DataSourceChannelFactory.java | 3 +- .../doris/DorisDataSourceChannelFactory.java | 2 +- .../hana/HanaDataSourceChannelFactory.java | 3 +- .../hive/HiveDataSourceChannelFactory.java | 3 +- .../k8s/K8sDataSourceChannelFactory.java | 3 +- .../k8s/param/K8sDataSourceProcessor.java | 2 +- .../KyuubiDataSourceChannelFactory.java | 3 +- .../param/KyuubiDataSourceProcessorTest.java | 2 +- .../mysql/MySQLDataSourceChannelFactory.java | 3 +- .../OceanBaseDataSourceChannelFactory.java | 3 +- .../OracleDataSourceChannelFactory.java | 3 +- .../PostgreSQLDataSourceChannelFactory.java | 3 +- .../PrestoDataSourceChannelFactory.java | 3 +- .../RedshiftDataSourceChannelFactory.java | 3 +- .../SagemakerDataSourceChannelFactory.java | 3 +- .../param/SagemakerDataSourceProcessor.java | 2 +- .../SnowflakeDataSourceChannelFactory.java | 3 +- .../SnowflakeDataSourceProcessorTest.java | 2 +- .../spark/SparkDataSourceChannelFactory.java | 3 +- .../SQLServerDataSourceChannelFactory.java | 3 +- .../ssh/SSHDataSourceChannelFactory.java | 3 +- .../ssh/param/SSHDataSourceProcessor.java | 2 +- .../StarRocksDataSourceChannelFactory.java | 2 +- .../trino/TrinoDataSourceChannelFactory.java | 3 +- .../VerticaDataSourceChannelFactory.java | 3 +- .../ZeppelinDataSourceChannelFactory.java | 3 +- .../param/ZeppelinDataSourceProcessor.java | 2 +- .../dolphinscheduler/spi/enums/DbType.java | 62 ++++++++++--------- 36 files changed, 95 insertions(+), 66 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index 98222a2c5a..4acf531ddc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -118,7 +118,7 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getUser(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getUser(), PasswordUtils.encodePassword(baseConnectionParam.getPassword()), baseConnectionParam.getJdbcUrl()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java index 839a4c5d61..7223fe62a3 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java @@ -69,9 +69,9 @@ public class DataSourceClientProvider { String datasourceUniqueId = DataSourceUtils.getDatasourceUniqueId(baseConnectionParam, dbType); return POOLED_DATASOURCE_CLIENT_CACHE.get(datasourceUniqueId, () -> { Map dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap(); - DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp()); + DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getName()); if (null == dataSourceChannel) { - throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getDescp())); + throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getName())); } return dataSourceChannel.createPooledDataSourceClient(baseConnectionParam, dbType); }); @@ -85,9 +85,9 @@ public class DataSourceClientProvider { public static AdHocDataSourceClient getAdHocDataSourceClient(DbType dbType, ConnectionParam connectionParam) { BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; Map dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap(); - DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp()); + DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getName()); if (null == dataSourceChannel) { - throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getDescp())); + throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getName())); } return dataSourceChannel.createAdHocDataSourceClient(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java index 1b2ed367d0..b4759db39a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.athena; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,6 +33,6 @@ public class AthenaDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "athena"; + return DbType.ATHENA.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java index 5966848f33..2b8cdca973 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.azuresql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class AzureSQLDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "azuresql"; + return DbType.AZURESQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java index d756226522..77d0feb1d1 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.clickhouse; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class ClickHouseDataSourceChannelFactory implements DataSourceChannelFact @Override public String getName() { - return "clickhouse"; + return DbType.CLICKHOUSE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java index 945f6610c0..84ae080134 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java @@ -28,7 +28,7 @@ public class DamengDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return DbType.DAMENG.getDescp(); + return DbType.DAMENG.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java index 0ea40c3b13..3c86601dd7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.databend; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class DatabendDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "databend"; + return DbType.DATABEND.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java index f225a2fd3d..cb41c6562b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java @@ -151,7 +151,7 @@ public class DatabendDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(19, DbType.DATABEND.getCode()); - Assertions.assertEquals("databend", DbType.DATABEND.getDescp()); + Assertions.assertEquals("databend", DbType.DATABEND.getName()); Assertions.assertEquals(DbType.DATABEND, DbType.of(19)); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java index cda8a2e592..3bbae238ea 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.db2; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class DB2DataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "db2"; + return DbType.DB2.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java index d663c362f2..7180a6c6c2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java @@ -32,6 +32,6 @@ public class DorisDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return DbType.DORIS.getDescp(); + return DbType.DORIS.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java index 75aacebaff..91d275aab6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hana; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class HanaDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "hana"; + return DbType.HANA.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java index 96ee007c8d..2caa4092dc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class HiveDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "hive"; + return DbType.HIVE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java index 03ec046de8..6a4428b47b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.k8s; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class K8sDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "k8s"; + return DbType.K8S.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java index 9e7342d433..fd3b49469f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java @@ -58,7 +58,7 @@ public class K8sDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { K8sConnectionParam baseConnectionParam = (K8sConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}", dbType.getDescp(), + return MessageFormat.format("{0}@{1}@{2}", dbType.getName(), PasswordUtils.encodePassword(baseConnectionParam.getKubeConfig()), baseConnectionParam.getNamespace()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java index 4c67a2098f..c60e74ccf8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.kyuubi; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class KyuubiDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "kyuubi"; + return DbType.KYUUBI.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java index 865565c5dc..a18ceb4216 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java @@ -143,7 +143,7 @@ public class KyuubiDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(18, DbType.KYUUBI.getCode()); - Assertions.assertEquals("kyuubi", DbType.KYUUBI.getDescp()); + Assertions.assertEquals("kyuubi", DbType.KYUUBI.getName()); Assertions.assertEquals(DbType.KYUUBI, DbType.of(18)); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java index e57fc7e61d..adc3ec7946 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.mysql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class MySQLDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "mysql"; + return DbType.MYSQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java index a69d6b3ae5..13650679b0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.oceanbase; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class OceanBaseDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "oceanbase"; + return DbType.OCEANBASE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java index dedbce4946..f63aff9a2b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.oracle; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class OracleDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "oracle"; + return DbType.ORACLE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java index 8aa6e566b7..e82a2e6860 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.postgresql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class PostgreSQLDataSourceChannelFactory implements DataSourceChannelFact @Override public String getName() { - return "postgresql"; + return DbType.POSTGRESQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java index ed1292ffc9..76bf9d0808 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.presto; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class PrestoDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "presto"; + return DbType.PRESTO.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java index 25a587ae06..8c588f0b44 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,6 +33,6 @@ public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "redshift"; + return DbType.REDSHIFT.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java index 04ab93f36f..2457843614 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.sagemaker; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class SagemakerDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "sagemaker"; + return DbType.SAGEMAKER.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java index 4239f45e5c..7452ef8a14 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java @@ -57,7 +57,7 @@ public class SagemakerDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { SagemakerConnectionParam baseConnectionParam = (SagemakerConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), PasswordUtils.encodePassword(baseConnectionParam.getUserName()), PasswordUtils.encodePassword(baseConnectionParam.getPassword()), PasswordUtils.encodePassword(baseConnectionParam.getAwsRegion())); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java index 0d0c97ecd6..6bbfd7a6fb 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.snowflake; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SnowflakeDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "snowflake"; + return DbType.SNOWFLAKE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java index 54c5acf0f2..c60e70576f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java @@ -169,7 +169,7 @@ public class SnowflakeDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(20, DbType.SNOWFLAKE.getCode()); - Assertions.assertEquals("snowflake", DbType.SNOWFLAKE.getDescp()); + Assertions.assertEquals("snowflake", DbType.SNOWFLAKE.getName()); Assertions.assertEquals(DbType.of(20), DbType.SNOWFLAKE); Assertions.assertEquals(DbType.ofName("SNOWFLAKE"), DbType.SNOWFLAKE); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java index dbda3da5bd..25f29ff21f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.spark; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SparkDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "spark"; + return DbType.SPARK.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java index e76f520d1e..f29cf6415e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.sqlserver; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SQLServerDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "sqlserver"; + return DbType.SQLSERVER.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java index 3195432703..9742c97e9b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.ssh; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SSHDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "ssh"; + return DbType.SSH.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java index 6bf0bed1b9..1916edba35 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java @@ -55,7 +55,7 @@ public class SSHDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { SSHConnectionParam baseConnectionParam = (SSHConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getHost(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getHost(), baseConnectionParam.getUser(), PasswordUtils.encodePassword(baseConnectionParam.getPassword())); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java index 50e2483952..82f78cff21 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java @@ -33,6 +33,6 @@ public class StarRocksDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return DbType.STARROCKS.getDescp(); + return DbType.STARROCKS.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java index 8c9605d791..36a3817fb0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.trino; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class TrinoDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "trino"; + return DbType.TRINO.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java index b507a207b4..44e151f2f2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.vertica; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class VerticaDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "vertica"; + return DbType.VERTICA.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java index 692819cf78..559ee55836 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.zeppelin; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class ZeppelinDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "zeppelin"; + return DbType.ZEPPELIN.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java index 92077275ad..88a913974e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java @@ -56,7 +56,7 @@ public class ZeppelinDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getRestEndpoint(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getRestEndpoint(), baseConnectionParam.getUsername(), PasswordUtils.encodePassword(baseConnectionParam.getPassword())); } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index e7ebbeee0a..882b170e11 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -28,42 +28,44 @@ import com.google.common.base.Functions; public enum DbType { - MYSQL(0, "mysql"), - POSTGRESQL(1, "postgresql"), - HIVE(2, "hive"), - SPARK(3, "spark"), - CLICKHOUSE(4, "clickhouse"), - ORACLE(5, "oracle"), - SQLSERVER(6, "sqlserver"), - DB2(7, "db2"), - PRESTO(8, "presto"), - H2(9, "h2"), - REDSHIFT(10, "redshift"), - ATHENA(11, "athena"), - TRINO(12, "trino"), - STARROCKS(13, "starrocks"), - AZURESQL(14, "azuresql"), - DAMENG(15, "dameng"), - OCEANBASE(16, "oceanbase"), - SSH(17, "ssh"), - KYUUBI(18, "kyuubi"), - DATABEND(19, "databend"), - SNOWFLAKE(20, "snowflake"), - VERTICA(21, "vertica"), - HANA(22, "hana"), - DORIS(23, "doris"), - ZEPPELIN(24, "zeppelin"), - SAGEMAKER(25, "sagemaker"), + MYSQL(0, "mysql", "mysql"), + POSTGRESQL(1, "postgresql", "postgresql"), + HIVE(2, "hive", "hive"), + SPARK(3, "spark", "spark"), + CLICKHOUSE(4, "clickhouse", "clickhouse"), + ORACLE(5, "oracle", "oracle"), + SQLSERVER(6, "sqlserver", "sqlserver"), + DB2(7, "db2", "db2"), + PRESTO(8, "presto", "presto"), + H2(9, "h2", "h2"), + REDSHIFT(10, "redshift", "redshift"), + ATHENA(11, "athena", "athena"), + TRINO(12, "trino", "trino"), + STARROCKS(13, "starrocks", "starrocks"), + AZURESQL(14, "azuresql", "azuresql"), + DAMENG(15, "dameng", "dameng"), + OCEANBASE(16, "oceanbase", "oceanbase"), + SSH(17, "ssh", "ssh"), + KYUUBI(18, "kyuubi", "kyuubi"), + DATABEND(19, "databend", "databend"), + SNOWFLAKE(20, "snowflake", "snowflake"), + VERTICA(21, "vertica", "vertica"), + HANA(22, "hana", "hana"), + DORIS(23, "doris", "doris"), + ZEPPELIN(24, "zeppelin", "zeppelin"), + SAGEMAKER(25, "sagemaker", "sagemaker"), - K8S(26, "k8s"); + K8S(26, "k8s", "k8s"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); @EnumValue private final int code; + private final String name; private final String descp; - DbType(int code, String descp) { + DbType(int code, String name, String descp) { this.code = code; + this.name = name; this.descp = descp; } @@ -83,6 +85,10 @@ public enum DbType { return code; } + public String getName() { + return name; + } + public String getDescp() { return descp; } From 5a6c6c37f006291bc774ec44fe6d3d6793721195 Mon Sep 17 00:00:00 2001 From: calvin Date: Sun, 28 Apr 2024 18:47:20 +0800 Subject: [PATCH 36/41] [Improvement-15910][UI] Supposed to provide a default value for the custom parallelism when using the mode of parallel execution. (#15912) * worked out the issue * imrpove the parallism strategy * imrpove the parallism strategy * merge from dev --- .../src/locales/en_US/project.ts | 3 ++- .../src/locales/zh_CN/project.ts | 3 ++- .../definition/components/start-modal.tsx | 20 ++++++++++--------- .../definition/components/use-form.ts | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 7a39752526..5e5ee4b2a9 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -249,7 +249,8 @@ export default { delete_task_validate_dependent_tasks_desc: 'The downstream dependent tasks exists. You can not delete the task.', warning_delete_scheduler_dependent_tasks_desc: - 'The downstream dependent tasks exists. Are you sure to delete the scheduler?' + 'The downstream dependent tasks exists. Are you sure to delete the scheduler?', + warning_too_large_parallelism_number: 'The parallelism number is too large. It is better not to be over 10.' }, task: { on_line: 'Online', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 50e0a821ef..85de0122fb 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -246,7 +246,8 @@ export default { delete_task_validate_dependent_tasks_desc: '下游存在依赖，你不能删除该任务.', warning_delete_scheduler_dependent_tasks_desc: - '下游存在依赖, 删除定时可能会对下游任务产生影响. 你确定要删除该定时嘛?' + '下游存在依赖, 删除定时可能会对下游任务产生影响. 你确定要删除该定时嘛?', + warning_too_large_parallelism_number: '并行度设置太大了, 最好不要超过10.', }, task: { on_line: '线上', diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx index 872b515844..8a4b07b4bc 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx @@ -44,7 +44,8 @@ import { NSwitch, NCheckbox, NDatePicker, - NRadioButton + NRadioButton, + NInputNumber } from 'naive-ui' import { ArrowDownOutlined, @@ -75,7 +76,6 @@ export default defineComponent({ props, emits: ['update:show', 'update:row', 'updateList'], setup(props, ctx) { - const parallelismRef = ref(false) const { t } = useI18n() const route = useRoute() const { startState } = useForm() @@ -296,7 +296,6 @@ export default defineComponent({ return { t, showTaskDependType, - parallelismRef, hideModal, handleStart, generalWarningTypeListOptions, @@ -504,17 +503,20 @@ export default defineComponent({ 10 + } > - - {t('project.workflow.custom_parallelism')} - - )} diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts index 32b02a7880..33dd7ba241 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts @@ -66,7 +66,7 @@ export const useForm = () => { tenantCode: 'default', environmentCode: null, startParams: null, - expectedParallelismNumber: '', + expectedParallelismNumber: '2', dryRun: 0, testFlag: 0, version: null, From 647cbae4002c0ab3758d57827460ad7125a2c853 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 29 Apr 2024 16:14:23 +0800 Subject: [PATCH 37/41] [DSIP-32][Master] Add command fetcher strategy for master fetch command (#15900) --- .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - docs/docs/en/architecture/configuration.md | 4 +- .../en/guide/installation/pseudo-cluster.md | 1 - docs/docs/zh/architecture/configuration.md | 47 +++++----- .../zh/guide/installation/pseudo-cluster.md | 1 - .../dao/mapper/CommandMapper.java | 12 +-- .../dao/repository/BaseDao.java | 5 ++ .../dao/repository/CommandDao.java | 39 ++++++++ .../dolphinscheduler/dao/repository/IDao.java | 5 ++ .../dao/repository/impl/CommandDaoImpl.java | 41 +++++++++ .../dao/mapper/CommandMapper.xml | 6 +- .../dao/mapper/CommandMapperTest.java | 13 ++- .../repository/impl/CommandDaoImplTest.java | 88 +++++++++++++++++++ .../command/CommandFetcherConfiguration.java | 49 +++++++++++ .../master/command/ICommandFetcher.java | 36 ++++++++ .../command/IdSlotBasedCommandFetcher.java | 73 +++++++++++++++ .../master/config/CommandFetchStrategy.java | 63 +++++++++++++ .../server/master/config/MasterConfig.java | 12 +-- .../runner/MasterSchedulerBootstrap.java | 38 ++------ .../src/main/resources/application.yaml | 9 +- .../master/config/MasterConfigTest.java | 12 +++ .../src/test/resources/application.yaml | 9 +- .../service/command/CommandService.java | 11 --- .../service/command/CommandServiceImpl.java | 9 -- .../command/MessageServiceImplTest.java | 10 --- .../src/main/resources/application.yaml | 9 +- 29 files changed, 484 insertions(+), 122 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh index 58937e740c..8536eb0905 100755 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456 # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh index 671c70a5bb..f64e59b768 100755 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456 # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh index e7fd1b7204..29f8570319 100644 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=jdbc diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh index 1dbd63254e..6851716058 100644 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 13d8932943..fe0b7851ba 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -286,7 +286,6 @@ Location: `master-server/conf/application.yaml` | Parameters | Default value | Description | |-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | master.listen-port | 5678 | master listen port | -| master.fetch-command-num | 10 | the number of commands fetched by master | | master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel | | master.exec-threads | 100 | master execute thread number to limit process instances in parallel | | master.dispatch-task-number | 3 | master dispatch task number per batch | @@ -305,6 +304,9 @@ Location: `master-server/conf/application.yaml` | master.registry-disconnect-strategy.strategy | stop | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting | | master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely | | master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory | +| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db | +| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master | ### Worker Server related configuration diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index e63436f203..7a3b43b00e 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -123,7 +123,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 08fded19e0..d8d1d42d1e 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -281,29 +281,30 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 位置：`master-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -|-----------------------------------------------------------------------------|--------------|-----------------------------------------------------------------------------------------| -| master.listen-port | 5678 | master监听端口 | -| master.fetch-command-num | 10 | master拉取command数量 | -| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | -| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | -| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | -| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | -| master.max-heartbeat-interval | 10s | master最大心跳间隔 | -| master.task-commit-retry-times | 5 | 任务重试次数 | -| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | -| master.state-wheel-interval | 5 | 轮询检查状态时间 | -| master.server-load-protection.enabled | true | 是否开启系统保护策略 | -| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | -| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | -| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | -| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | -| master.failover-interval | 10 | failover间隔，单位为分钟 | -| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | -| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | -| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, | -| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | -| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| 参数 | 默认值 | 描述 | +|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------| +| master.listen-port | 5678 | master监听端口 | +| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | +| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | +| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | +| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | +| master.max-heartbeat-interval | 10s | master最大心跳间隔 | +| master.task-commit-retry-times | 5 | 任务重试次数 | +| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | +| master.state-wheel-interval | 5 | 轮询检查状态时间 | +| master.server-load-protection.enabled | true | 是否开启系统保护策略 | +| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | +| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | +| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | +| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | +| master.failover-interval | 10 | failover间隔，单位为分钟 | +| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | +| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | +| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | +| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| master.command-fetch-strategy.type | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | 数据库中t_ds_command的id自增步长 | +| master.command-fetch-strategy.config.fetch-size | 10 | master拉取command数量 | ## Worker Server相关配置 diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 13479e0d9e..a199167e04 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -118,7 +118,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index 9fb6643227..8c8314e7cc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -52,14 +52,10 @@ public interface CommandMapper extends BaseMapper { */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); - /** - * query command page by slot - * - * @return command list - */ - List queryCommandPageBySlot(@Param("limit") int limit, - @Param("masterCount") int masterCount, - @Param("thisMasterSlot") int thisMasterSlot); + List queryCommandByIdSlot(@Param("currentSlotIndex") int currentSlotIndex, + @Param("totalSlot") int totalSlot, + @Param("idStep") int idStep, + @Param("fetchNumber") int fetchNum); void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java index 2937957dbd..664b56ee47 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java @@ -56,6 +56,11 @@ public abstract class BaseDao> return mybatisMapper.selectBatchIds(ids); } + @Override + public List queryAll() { + return mybatisMapper.selectList(null); + } + @Override public List queryByCondition(ENTITY queryCondition) { if (queryCondition == null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java new file mode 100644 index 0000000000..daa52b8318 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.dao.entity.Command; + +import java.util.List; + +public interface CommandDao extends IDao { + + /** + * Query command by command id and server slot, return the command which match (commandId / step) %s totalSlot = currentSlotIndex + * + * @param currentSlotIndex current slot index + * @param totalSlot total slot number + * @param idStep id step in db + * @param fetchNum fetch number + * @return command list + */ + List queryCommandByIdSlot(int currentSlotIndex, + int totalSlot, + int idStep, + int fetchNum); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java index c566d9b904..ab77419600 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java @@ -41,6 +41,11 @@ public interface IDao { */ List queryByIds(Collection ids); + /** + * Query all entities. + */ + List queryAll(); + /** * Query the entity by condition. */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java new file mode 100644 index 0000000000..0b510d15b5 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import java.util.List; + +import org.springframework.stereotype.Repository; + +@Repository +public class CommandDaoImpl extends BaseDao implements CommandDao { + + public CommandDaoImpl(CommandMapper commandMapper) { + super(commandMapper); + } + + @Override + public List queryCommandByIdSlot(int currentSlotIndex, int totalSlot, int idStep, int fetchNum) { + return mybatisMapper.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchNum); + } + +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index 56db890ef0..16f7c05f25 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -40,12 +40,12 @@ limit #{limit} offset #{offset} - select * from t_ds_command - where id % #{masterCount} = #{thisMasterSlot} + where (id / #{idStep}) % #{totalSlot} = #{currentSlotIndex} order by process_instance_priority, id asc - limit #{limit} + limit #{fetchNumber} delete from t_ds_command diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 2d367e46e4..560b68754a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -187,7 +187,7 @@ public class CommandMapperTest extends BaseDaoTest { Command command = createCommand(); Integer id = command.getId(); boolean hit = id % masterCount == thisMasterSlot; - List commandList = commandMapper.queryCommandPageBySlot(1, masterCount, thisMasterSlot); + List commandList = commandMapper.queryCommandByIdSlot(thisMasterSlot, masterCount, 1, 1); if (hit) { Assertions.assertEquals(id, commandList.get(0).getId()); } else { @@ -201,8 +201,9 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command map - * @param count map count - * @param commandType comman type + * + * @param count map count + * @param commandType comman type * @param processDefinitionCode process definition code * @return command map */ @@ -223,7 +224,8 @@ public class CommandMapperTest extends BaseDaoTest { } /** - * create process definition + * create process definition + * * @return process definition */ private ProcessDefinition createProcessDefinition() { @@ -243,6 +245,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command map + * * @param count map count * @return command map */ @@ -258,6 +261,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command + * * @return */ private Command createCommand() { @@ -266,6 +270,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command + * * @return Command */ private Command createCommand(CommandType commandType, long processDefinitionCode) { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java new file mode 100644 index 0000000000..85867ef3b5 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import org.apache.commons.lang3.RandomUtils; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class CommandDaoImplTest extends BaseDaoTest { + + @Autowired + private CommandDao commandDao; + + @Test + void fetchCommandByIdSlot() { + int commandSize = RandomUtils.nextInt(1, 1000); + for (int i = 0; i < commandSize; i++) { + createCommand(CommandType.START_PROCESS, 0); + } + int totalSlot = RandomUtils.nextInt(1, 10); + int currentSlotIndex = RandomUtils.nextInt(0, totalSlot); + int fetchSize = RandomUtils.nextInt(10, 100); + for (int i = 1; i < 5; i++) { + int idStep = i; + List commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize); + assertThat(commands.size()).isGreaterThan(0); + assertThat(commands.size()) + .isEqualTo(commandDao.queryAll() + .stream() + .filter(command -> (command.getId() / idStep) % totalSlot == currentSlotIndex) + .limit(fetchSize) + .count()); + + } + + } + + private void createCommand(CommandType commandType, int processDefinitionCode) { + Command command = new Command(); + command.setCommandType(commandType); + command.setProcessDefinitionCode(processDefinitionCode); + command.setExecutorId(4); + command.setCommandParam("test command param"); + command.setTaskDependType(TaskDependType.TASK_ONLY); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00")); + command.setProcessInstancePriority(Priority.MEDIUM); + command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP); + command.setProcessInstanceId(0); + command.setProcessDefinitionVersion(0); + commandDao.insert(command); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java new file mode 100644 index 0000000000..4a4d3c1efc --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.server.master.config.CommandFetchStrategy; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class CommandFetcherConfiguration { + + @Bean + public ICommandFetcher commandFetcher(MasterConfig masterConfig, + MasterSlotManager masterSlotManager, + CommandDao commandDao) { + CommandFetchStrategy commandFetchStrategy = + checkNotNull(masterConfig.getCommandFetchStrategy(), "command fetch strategy is null"); + switch (commandFetchStrategy.getType()) { + case ID_SLOT_BASED: + CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig = + (CommandFetchStrategy.IdSlotBasedFetchConfig) commandFetchStrategy.getConfig(); + return new IdSlotBasedCommandFetcher(idSlotBasedFetchConfig, masterSlotManager, commandDao); + default: + throw new IllegalArgumentException( + "unsupported command fetch strategy type: " + commandFetchStrategy.getType()); + } + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java new file mode 100644 index 0000000000..c315a9b294 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import org.apache.dolphinscheduler.dao.entity.Command; + +import java.util.List; + +/** + * The command fetcher used to fetch commands + */ +public interface ICommandFetcher { + + /** + * Fetch commands + * + * @return command list which need to be handled + */ + List fetchCommands(); + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java new file mode 100644 index 0000000000..a417820093 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.server.master.config.CommandFetchStrategy; +import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; +import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; + +import java.util.Collections; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +/** + * The command fetcher which is fetch commands by command id and slot. + */ +@Slf4j +public class IdSlotBasedCommandFetcher implements ICommandFetcher { + + private final CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig; + + private final CommandDao commandDao; + + private final MasterSlotManager masterSlotManager; + + public IdSlotBasedCommandFetcher(CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig, + MasterSlotManager masterSlotManager, + CommandDao commandDao) { + this.idSlotBasedFetchConfig = idSlotBasedFetchConfig; + this.masterSlotManager = masterSlotManager; + this.commandDao = commandDao; + } + + @Override + public List fetchCommands() { + long scheduleStartTime = System.currentTimeMillis(); + int currentSlotIndex = masterSlotManager.getSlot(); + int totalSlot = masterSlotManager.getMasterSize(); + if (totalSlot <= 0 || currentSlotIndex < 0) { + log.warn("Slot is validated, current master slots: {}, the current slot index is {}", totalSlot, + currentSlotIndex); + return Collections.emptyList(); + } + List commands = commandDao.queryCommandByIdSlot( + currentSlotIndex, + totalSlot, + idSlotBasedFetchConfig.getIdStep(), + idSlotBasedFetchConfig.getFetchSize()); + long cost = System.currentTimeMillis() - scheduleStartTime; + log.info("Fetch commands: {} success, cost: {}ms, totalSlot: {}, currentSlotIndex: {}", commands.size(), cost, + totalSlot, currentSlotIndex); + ProcessInstanceMetrics.recordCommandQueryTime(cost); + return commands; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java new file mode 100644 index 0000000000..e61941677c --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.config; + +import lombok.Data; + +import org.springframework.validation.Errors; + +@Data +public class CommandFetchStrategy { + + private CommandFetchStrategyType type = CommandFetchStrategyType.ID_SLOT_BASED; + + private CommandFetchConfig config = new IdSlotBasedFetchConfig(); + + public void validate(Errors errors) { + config.validate(errors); + } + + public enum CommandFetchStrategyType { + ID_SLOT_BASED, + ; + } + + public interface CommandFetchConfig { + + void validate(Errors errors); + + } + + @Data + public static class IdSlotBasedFetchConfig implements CommandFetchConfig { + + private int idStep = 1; + private int fetchSize = 10; + + @Override + public void validate(Errors errors) { + if (idStep <= 0) { + errors.rejectValue("step", null, "step must be greater than 0"); + } + if (fetchSize <= 0) { + errors.rejectValue("fetchSize", null, "fetchSize must be greater than 0"); + } + } + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index 02c0dcb819..20d3cccef3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -48,10 +48,6 @@ public class MasterConfig implements Validator { * The master RPC server listen port. */ private int listenPort = 5678; - /** - * The max batch size used to fetch command from database. - */ - private int fetchCommandNum = 10; /** * The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum. */ @@ -98,6 +94,8 @@ public class MasterConfig implements Validator { private Duration workerGroupRefreshInterval = Duration.ofSeconds(10L); + private CommandFetchStrategy commandFetchStrategy = new CommandFetchStrategy(); + // ip:listenPort private String masterAddress; @@ -115,9 +113,6 @@ public class MasterConfig implements Validator { if (masterConfig.getListenPort() <= 0) { errors.rejectValue("listen-port", null, "is invalidated"); } - if (masterConfig.getFetchCommandNum() <= 0) { - errors.rejectValue("fetch-command-num", null, "should be a positive value"); - } if (masterConfig.getPreExecThreads() <= 0) { errors.rejectValue("per-exec-threads", null, "should be a positive value"); } @@ -149,6 +144,7 @@ public class MasterConfig implements Validator { if (StringUtils.isEmpty(masterConfig.getMasterAddress())) { masterConfig.setMasterAddress(NetUtils.getAddr(masterConfig.getListenPort())); } + commandFetchStrategy.validate(errors); masterConfig.setMasterRegistryPath( RegistryNodeType.MASTER.getRegistryPath() + "/" + masterConfig.getMasterAddress()); @@ -159,7 +155,6 @@ public class MasterConfig implements Validator { String config = "\n****************************Master Configuration**************************************" + "\n listen-port -> " + listenPort + - "\n fetch-command-num -> " + fetchCommandNum + "\n pre-exec-threads -> " + preExecThreads + "\n exec-threads -> " + execThreads + "\n dispatch-task-number -> " + dispatchTaskNumber + @@ -175,6 +170,7 @@ public class MasterConfig implements Validator { "\n master-address -> " + masterAddress + "\n master-registry-path: " + masterRegistryPath + "\n worker-group-refresh-interval: " + workerGroupRefreshInterval + + "\n command-fetch-strategy: " + commandFetchStrategy + "\n****************************Master Configuration**************************************"; log.info(config); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java index 2fddd94384..c1b5d0ffab 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java @@ -26,21 +26,18 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; +import org.apache.dolphinscheduler.server.master.command.ICommandFetcher; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection; import org.apache.dolphinscheduler.server.master.event.WorkflowEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.event.WorkflowEventType; -import org.apache.dolphinscheduler.server.master.exception.MasterException; import org.apache.dolphinscheduler.server.master.exception.WorkflowCreateException; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; -import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; -import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.commons.collections4.CollectionUtils; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -56,6 +53,9 @@ import org.springframework.stereotype.Service; @Slf4j public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCloseable { + @Autowired + private ICommandFetcher commandFetcher; + @Autowired private CommandService commandService; @@ -74,9 +74,6 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl @Autowired private WorkflowEventLooper workflowEventLooper; - @Autowired - private MasterSlotManager masterSlotManager; - @Autowired private MasterTaskExecutorBootstrap masterTaskExecutorBootstrap; @@ -125,7 +122,7 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } - List commands = findCommands(); + List commands = commandFetcher.fetchCommands(); if (CollectionUtils.isEmpty(commands)) { // indicate that no command ,sleep for 1s Thread.sleep(Constants.SLEEP_TIME_MILLIS); @@ -170,29 +167,4 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl } } - private List findCommands() throws MasterException { - try { - long scheduleStartTime = System.currentTimeMillis(); - int thisMasterSlot = masterSlotManager.getSlot(); - int masterCount = masterSlotManager.getMasterSize(); - if (masterCount <= 0) { - log.warn("Master count: {} is invalid, the current slot: {}", masterCount, thisMasterSlot); - return Collections.emptyList(); - } - int pageSize = masterConfig.getFetchCommandNum(); - final List result = - commandService.findCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - if (CollectionUtils.isNotEmpty(result)) { - long cost = System.currentTimeMillis() - scheduleStartTime; - log.info( - "Master schedule bootstrap loop command success, fetch command size: {}, cost: {}ms, current slot: {}, total slot size: {}", - result.size(), cost, thisMasterSlot, masterCount); - ProcessInstanceMetrics.recordCommandQueryTime(cost); - } - return result; - } catch (Exception ex) { - throw new MasterException("Master loop command from database error", ex); - } - } - } diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index f18c6ef61d..17b1e41a71 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -83,8 +83,6 @@ registry: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -121,6 +119,13 @@ master: # The max waiting time to reconnect to registry if you set the strategy to waiting max-waiting-time: 100s worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 server: port: 5679 diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java index faab44cf85..9d26aa81f4 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.config; +import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -47,6 +48,17 @@ public class MasterConfigTest { assertEquals(0.77, serverLoadProtection.getMaxJvmCpuUsagePercentageThresholds()); assertEquals(0.77, serverLoadProtection.getMaxSystemMemoryUsagePercentageThresholds()); assertEquals(0.77, serverLoadProtection.getMaxDiskUsagePercentageThresholds()); + } + @Test + public void getCommandFetchStrategy() { + CommandFetchStrategy commandFetchStrategy = masterConfig.getCommandFetchStrategy(); + assertThat(commandFetchStrategy.getType()) + .isEqualTo(CommandFetchStrategy.CommandFetchStrategyType.ID_SLOT_BASED); + + CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig = + (CommandFetchStrategy.IdSlotBasedFetchConfig) commandFetchStrategy.getConfig(); + assertThat(idSlotBasedFetchConfig.getIdStep()).isEqualTo(3); + assertThat(idSlotBasedFetchConfig.getFetchSize()).isEqualTo(11); } } diff --git a/dolphinscheduler-master/src/test/resources/application.yaml b/dolphinscheduler-master/src/test/resources/application.yaml index f4827d4b3c..15f9199609 100644 --- a/dolphinscheduler-master/src/test/resources/application.yaml +++ b/dolphinscheduler-master/src/test/resources/application.yaml @@ -89,8 +89,6 @@ registry: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -127,6 +125,13 @@ master: # The max waiting time to reconnect to registry if you set the strategy to waiting max-waiting-time: 100s worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 3 + # master fetch command num + fetch-size: 11 server: port: 5679 diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java index cff73c503f..43b81c4e5c 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java @@ -22,8 +22,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import java.util.List; - /** * Command Service */ @@ -44,15 +42,6 @@ public interface CommandService { */ int createCommand(Command command); - /** - * Get command page - * @param pageSize page size - * @param masterCount master count - * @param thisMasterSlot master slot - * @return command page - */ - List findCommandPageBySlot(int pageSize, int masterCount, int thisMasterSlot); - /** * check the input command exists in queue list * diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java index 483899446b..ee833a80b0 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java @@ -57,7 +57,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; import io.micrometer.core.annotation.Counted; /** @@ -107,14 +106,6 @@ public class CommandServiceImpl implements CommandService { return result; } - @Override - public List findCommandPageBySlot(int pageSize, int masterCount, int thisMasterSlot) { - if (masterCount <= 0) { - return Lists.newArrayList(); - } - return commandMapper.queryCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - } - @Override public boolean verifyIsNeedCreateCommand(Command command) { boolean isNeedCreate = true; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java index 0cde76bdfe..f60320fc63 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java @@ -214,14 +214,4 @@ class MessageServiceImplTest { Mockito.verify(commandMapper, Mockito.times(1)).insert(command); } - @Test - public void testFindCommandPageBySlot() { - int pageSize = 1; - int masterCount = 0; - int thisMasterSlot = 2; - List commandList = - commandService.findCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - Assertions.assertEquals(0, commandList.size()); - } - } diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 5122eea2a1..6757718929 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -160,8 +160,6 @@ casdoor: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -192,6 +190,13 @@ master: # kill yarn/k8s application when failover taskInstance, default true kill-application-when-task-failover: true worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 worker: # worker listener port From 0a11cd21bd0e73b52d4e68dae2f32519031e2e50 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 29 Apr 2024 18:07:05 +0800 Subject: [PATCH 38/41] Fix jar path is not correct in java task (#15906) --- .../task/api/resource/ResourceContext.java | 1 - .../plugin/task/hivecli/HiveCliTaskTest.java | 4 +- .../plugin/task/java/JavaTask.java | 41 ++----------------- .../plugin/task/java/JavaTaskTest.java | 32 ++++++++------- .../utils/TaskExecutionContextUtils.java | 1 - 5 files changed, 23 insertions(+), 56 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java index 687d1aeb95..f90b526902 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java @@ -60,7 +60,6 @@ public class ResourceContext { public static class ResourceItem { private String resourceAbsolutePathInStorage; - private String resourceRelativePath; private String resourceAbsolutePathInLocal; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java index 824ad49f89..b4136af3c3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java @@ -65,7 +65,7 @@ public class HiveCliTaskTest { } @Test - public void hiveCliTaskExecuteSqlFromScript() throws Exception { + public void hiveCliTaskExecuteSqlFromScript() { String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters(); HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); hiveCliTask.init(); @@ -78,7 +78,7 @@ public class HiveCliTaskTest { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskParams(hiveCliTaskParameters); ResourceContext resourceContext = new ResourceContext(); - resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql", + resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "/sql_tasks/hive_task.sql")); taskExecutionContext.setResourceContext(resourceContext); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index 179b50c35c..fc23260345 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -88,6 +88,7 @@ public class JavaTask extends AbstractTask { /** * Initializes a Java task + * * @return void **/ @Override @@ -178,7 +179,8 @@ public class JavaTask extends AbstractTask { **/ protected String buildJarCommand() { ResourceContext resourceContext = taskRequest.getResourceContext(); - String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName()) + String mainJarAbsolutePathInLocal = resourceContext + .getResourceItem(javaParameters.getMainJar().getResourceName()) .getResourceAbsolutePathInLocal(); StringBuilder builder = new StringBuilder(); builder.append(getJavaCommandPath()) @@ -186,7 +188,7 @@ public class JavaTask extends AbstractTask { .append(buildResourcePath()).append(" ") .append("-jar").append(" ") .append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR) - .append(mainJarName).append(" ") + .append(mainJarAbsolutePathInLocal).append(" ") .append(javaParameters.getMainArgs().trim()).append(" ") .append(javaParameters.getJvmArgs().trim()); return builder.toString(); @@ -207,39 +209,6 @@ public class JavaTask extends AbstractTask { return javaParameters; } - /** - * Replaces placeholders such as local variables in source files - * - * @param rawScript - * @return String - * @throws StringIndexOutOfBoundsException - */ - protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException { - int len = "${setShareVar(${".length(); - - int scriptStart = 0; - while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { - int start = -1; - int end = rawScript.indexOf('}', scriptStart + len); - String prop = rawScript.substring(scriptStart + len, end); - - start = rawScript.indexOf(',', end); - end = rawScript.indexOf(')', start); - - String value = rawScript.substring(start + 1, end); - - start = rawScript.indexOf('}', start) + 1; - end = rawScript.length(); - - String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); - - rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); - - scriptStart += replaceScript.length(); - } - return rawScript; - } - /** * Creates a Java source file when it does not exist * @@ -290,8 +259,6 @@ public class JavaTask extends AbstractTask { for (ResourceInfo info : javaParameters.getResourceFilesList()) { builder.append(JavaConstants.PATH_SEPARATOR); builder - .append(taskRequest.getExecutePath()) - .append(FOLDER_SEPARATOR) .append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal()); } return builder.toString(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java index 55756241ce..c829415326 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.java; +import static com.google.common.truth.Truth.assertThat; import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; @@ -34,7 +35,6 @@ import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExis import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; -import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; @@ -82,10 +82,10 @@ public class JavaTaskTest { **/ @Test public void buildJarCommand() { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; JavaTask javaTask = runJarType(); - Assertions.assertEquals(javaTask.buildJarCommand(), homeBinPath - + "java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); + assertThat(javaTask.buildJarCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); } /** @@ -101,14 +101,13 @@ public class JavaTaskTest { Assertions.assertEquals("JavaTaskTest", publicClassName); String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); try { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(homeBinPath - + "javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java", - javaTask.buildJavaCompileCommand(sourceCode)); + assertThat(javaTask.buildJavaCompileCommand(sourceCode)) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java"); } finally { Path path = Paths.get(fileName); if (Files.exists(path)) { @@ -121,26 +120,29 @@ public class JavaTaskTest { /** * Construct java to run the command * - * @return void + * @return void **/ @Test public void buildJavaCommand() throws Exception { - String wantJavaCommand = - "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"; JavaTask javaTask = runJavaType(); String sourceCode = javaTask.buildJavaSourceContent(); String publicClassName = javaTask.getPublicClassName(sourceCode); + Assertions.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(wantJavaCommand, javaTask.buildJavaCommand()); + assertThat(javaTask.buildJavaCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"); } /** * There is no exception to overwriting the Java source file + * * @return void * @throws IOException **/ @@ -259,8 +261,8 @@ public class JavaTaskTest { resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); - resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); - resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInLocal("JavaTaskTest.java"); ResourceContext resourceContext = new ResourceContext(); resourceContext.addResourceItem(resourceItem1); @@ -275,7 +277,7 @@ public class JavaTaskTest { /** * The Java task to construct the jar run mode * - * @return JavaTask + * @return JavaTask **/ private JavaTask runJarType() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index f43b19c444..8d83dde593 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -151,7 +151,6 @@ public class TaskExecutionContextUtils { } ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder() .resourceAbsolutePathInStorage(resourceAbsolutePathInStorage) - .resourceRelativePath(resourceRelativePath) .resourceAbsolutePathInLocal(resourceAbsolutePathInLocal) .build(); resourceContext.addResourceItem(resourceItem); From ebcdaeb9ac125a13b74ec3f057693816a6758426 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 29 Apr 2024 21:03:01 +0800 Subject: [PATCH 39/41] [TEST] increase coverage of project preference service test (#15939) --- .../service/ProjectPreferenceServiceTest.java | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java index 530c15d48e..7a74a7c265 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java @@ -60,28 +60,65 @@ public class ProjectPreferenceServiceTest { public void testUpdateProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + + // success Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); - Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(1); - Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operatation fail + Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.CREATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + + // success + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operation fail + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); } @Test public void testQueryProjectPreferenceByProjectCode() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + // PROJECT_PARAMETER_NOT_EXISTS Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), Mockito.any())).thenReturn(true); Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); - Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); // SUCCESS @@ -94,14 +131,29 @@ public class ProjectPreferenceServiceTest { public void testEnableProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); + // success Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); - Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + // db operation fail + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_STATE_ERROR.getCode(), result.getCode()); } private User getGeneralUser() { From fa6ea8ba7b06e21177114058b1f1c32fe005a60a Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 6 May 2024 10:02:04 +0800 Subject: [PATCH 40/41] [TEST] increase coverage of project workergroup relation service (#15944) Co-authored-by: abzymeinsjtu --- ...ProjectWorkerGroupRelationServiceImpl.java | 12 +- ...ProjectWorkerGroupRelationServiceTest.java | 108 ++++++++++++++---- 2 files changed, 96 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java index 15cbcbb7fa..31dc113dbb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectWorkerGroupMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; @@ -38,6 +39,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -113,23 +115,25 @@ public class ProjectWorkerGroupRelationServiceImpl extends BaseServiceImpl } Set workerGroupNames = - workerGroupMapper.queryAllWorkerGroup().stream().map(item -> item.getName()).collect( + workerGroupMapper.queryAllWorkerGroup().stream().map(WorkerGroup::getName).collect( Collectors.toSet()); workerGroupNames.add(Constants.DEFAULT_WORKER_GROUP); - Set assignedWorkerGroupNames = workerGroups.stream().collect(Collectors.toSet()); + Set assignedWorkerGroupNames = new HashSet<>(workerGroups); Set difference = SetUtils.difference(assignedWorkerGroupNames, workerGroupNames); - if (difference.size() > 0) { + if (!difference.isEmpty()) { putMsg(result, Status.WORKER_GROUP_NOT_EXIST, difference.toString()); return result; } Set projectWorkerGroupNames = projectWorkerGroupMapper.selectList(new QueryWrapper() .lambda() - .eq(ProjectWorkerGroup::getProjectCode, projectCode)).stream().map(item -> item.getWorkerGroup()) + .eq(ProjectWorkerGroup::getProjectCode, projectCode)) + .stream() + .map(ProjectWorkerGroup::getWorkerGroup) .collect(Collectors.toSet()); difference = SetUtils.difference(projectWorkerGroupNames, assignedWorkerGroupNames); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java index a8e8b8beb2..6f6b7ca2d6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java @@ -17,13 +17,17 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getAdminUser; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectWorkerGroupRelationServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -33,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -77,27 +82,87 @@ public class ProjectWorkerGroupRelationServiceTest { @Test public void testAssignWorkerGroupsToProject() { + User generalUser = getGeneralUser(); User loginUser = getAdminUser(); - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); - Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + // no permission + Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(generalUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), result.getCode()); + + // project code is null + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, null, getWorkerGroups()); Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + // worker group is empty + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + Collections.emptyList()); + Assertions.assertEquals(Status.WORKER_GROUP_TO_PROJECT_IS_EMPTY.getCode(), result.getCode()); + + // project not exists + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + + // worker group not exists WorkerGroup workerGroup = new WorkerGroup(); workerGroup.setName("test"); Mockito.when(projectMapper.queryByCode(Mockito.anyLong())).thenReturn(getProject()); - Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Lists.newArrayList(workerGroup)); + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getDiffWorkerGroups()); + Assertions.assertEquals(Status.WORKER_GROUP_NOT_EXIST.getCode(), result.getCode()); + + // db insertion fail + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // success Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(1); result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, getWorkerGroups()); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // success when there is diff between current wg and assigned wg + Mockito.when(projectWorkerGroupMapper.selectList(Mockito.any())) + .thenReturn(Collections.singletonList(getDiffProjectWorkerGroup())); + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(1); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // db deletion fail + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // fail when wg is referenced by task definition + Mockito.when(taskDefinitionMapper.queryAllDefinitionList(Mockito.anyLong())) + .thenReturn(Collections.singletonList(getTaskDefinitionWithDiffWorkerGroup())); + AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); } @Test public void testQueryWorkerGroupsByProject() { + // no permission + Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) + .thenReturn(false); + Map result = + projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + + Assertions.assertTrue(result.isEmpty()); + + // success Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) .thenReturn(true); @@ -113,8 +178,7 @@ public class ProjectWorkerGroupRelationServiceTest { Mockito.when(scheduleMapper.querySchedulerListByProjectName(Mockito.any())) .thenReturn(Lists.newArrayList()); - Map result = - projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + result = projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); ProjectWorkerGroup[] actualValue = ((List) result.get(Constants.DATA_LIST)).toArray(new ProjectWorkerGroup[0]); @@ -126,20 +190,8 @@ public class ProjectWorkerGroupRelationServiceTest { return Lists.newArrayList("default"); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; - } - - private User getAdminUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.ADMIN_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + private List getDiffWorkerGroups() { + return Lists.newArrayList("default", "new"); } private Project getProject() { @@ -158,4 +210,20 @@ public class ProjectWorkerGroupRelationServiceTest { projectWorkerGroup.setWorkerGroup("default"); return projectWorkerGroup; } + + private ProjectWorkerGroup getDiffProjectWorkerGroup() { + ProjectWorkerGroup projectWorkerGroup = new ProjectWorkerGroup(); + projectWorkerGroup.setId(2); + projectWorkerGroup.setProjectCode(projectCode); + projectWorkerGroup.setWorkerGroup("new"); + return projectWorkerGroup; + } + + private TaskDefinition getTaskDefinitionWithDiffWorkerGroup() { + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setProjectCode(projectCode); + taskDefinition.setId(1); + taskDefinition.setWorkerGroup("new"); + return taskDefinition; + } } From d218b021e7e045e7982a3c91ea7b1b1491d21d38 Mon Sep 17 00:00:00 2001 From: JohnHuang Date: Mon, 6 May 2024 11:33:16 +0800 Subject: [PATCH 41/41] [DSIP-25][Remote Logging] Split remote logging configuration (#15826) Co-authored-by: Rick Cheng --- .../dolphinscheduler-alert-server.xml | 1 + .../assembly/dolphinscheduler-api-server.xml | 1 + .../ImmutablePriorityPropertyDelegate.java | 33 ++++++-- .../config/ImmutablePropertyDelegate.java | 2 +- .../common/config/ImmutableYamlDelegate.java | 82 +++++++++++++++++++ .../common/constants/Constants.java | 5 +- .../common/utils/PropertyUtils.java | 10 ++- .../src/main/resources/remote-logging.yaml | 61 ++++++++++++++ ...ImmutablePriorityPropertyDelegateTest.java | 5 +- .../src/test/resources/remote-logging.yaml | 61 ++++++++++++++ .../dolphinscheduler-master-server.xml | 1 + .../dolphinscheduler-worker-server.xml | 1 + 12 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java create mode 100644 dolphinscheduler-common/src/main/resources/remote-logging.yaml create mode 100644 dolphinscheduler-common/src/test/resources/remote-logging.yaml diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 6f39096221..bf28193b3f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -52,6 +52,7 @@ ${basedir}/../../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index 77b2f54c64..c9fdab4d51 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java index 742e745fe4..620f74ef95 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java @@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j; * This class will get the property by the priority of the following: env > jvm > properties. */ @Slf4j -public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate { +public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate { private static final Map>> configValueMap = new ConcurrentHashMap<>(); - public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) { - super(propertyAbsolutePath); + private ImmutablePropertyDelegate immutablePropertyDelegate; + + private ImmutableYamlDelegate immutableYamlDelegate; + + public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate, + ImmutableYamlDelegate immutableYamlDelegate) { + this.immutablePropertyDelegate = immutablePropertyDelegate; + this.immutableYamlDelegate = immutableYamlDelegate; } @Override @@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate return value; } value = getConfigValueFromProperties(key); + if (value.isPresent()) { + log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + k, value.get().getActualKey(), value.get().getValue()); + return value; + } + value = getConfigValueFromYaml(key); value.ifPresent( - stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}", k, stringConfigValue.getActualKey(), stringConfigValue.getValue())); return value; }); @@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate @Override public Set getPropertyKeys() { Set propertyKeys = new HashSet<>(); - propertyKeys.addAll(super.getPropertyKeys()); + propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys()); + propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys()); propertyKeys.addAll(System.getProperties().stringPropertyNames()); propertyKeys.addAll(System.getenv().keySet()); return propertyKeys; @@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate } private Optional> getConfigValueFromProperties(String key) { - String value = super.get(key); + String value = this.immutablePropertyDelegate.get(key); + if (value != null) { + return Optional.of(ConfigValue.fromProperties(key, value)); + } + return Optional.empty(); + } + + private Optional> getConfigValueFromYaml(String key) { + String value = this.immutableYamlDelegate.get(key); if (value != null) { return Optional.of(ConfigValue.fromProperties(key, value)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java index b58735afb0..4a0c192210 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java @@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate { } catch (IOException e) { log.error("Load property: {} error, please check if the file exist under classpath", propertyAbsolutePath, e); - System.exit(1); + throw new RuntimeException(e); } } printProperties(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java new file mode 100644 index 0000000000..5806a20fd7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.config; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.InputStreamResource; + +@Slf4j +public class ImmutableYamlDelegate implements IPropertyDelegate { + + private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml"; + + private final Properties properties; + + public ImmutableYamlDelegate() { + this(REMOTE_LOGGING_YAML_NAME); + } + + public ImmutableYamlDelegate(String... yamlAbsolutePath) { + properties = new Properties(); + // read from classpath + for (String fileName : yamlAbsolutePath) { + try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { + YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); + factory.setResources(new InputStreamResource(fis)); + factory.afterPropertiesSet(); + Properties subProperties = factory.getObject(); + properties.putAll(subProperties); + } catch (IOException e) { + log.error("Load property: {} error, please check if the file exist under classpath", + yamlAbsolutePath, e); + throw new RuntimeException(e); + } + } + printProperties(); + } + + public ImmutableYamlDelegate(Properties properties) { + this.properties = properties; + } + + @Override + public String get(String key) { + return properties.getProperty(key); + } + + @Override + public String get(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } + + @Override + public Set getPropertyKeys() { + return properties.stringPropertyNames(); + } + + private void printProperties() { + properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v)); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index ebf668a312..cc07accc9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -35,6 +35,8 @@ public final class Constants { */ public static final String COMMON_PROPERTIES_PATH = "/common.properties"; + public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; + public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; public static final String FORMAT_S_S_COLON = "%s:%s"; @@ -683,9 +685,6 @@ public final class Constants { public static final Integer QUERY_ALL_ON_WORKFLOW = 2; public static final Integer QUERY_ALL_ON_TASK = 3; - /** - * remote logging - */ public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable"; public static final String REMOTE_LOGGING_TARGET = "remote.logging.target"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8289b14479..82d4de9599 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; -import org.apache.dolphinscheduler.common.config.IPropertyDelegate; import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate; import java.util.HashMap; import java.util.Map; @@ -37,8 +39,10 @@ import com.google.common.base.Strings; public class PropertyUtils { // todo: add another implementation for zookeeper/etcd/consul/xx - private static final IPropertyDelegate propertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + private final ImmutablePriorityPropertyDelegate propertyDelegate = + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml new file mode 100644 index 0000000000..2cb48750a4 --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java index efba923a5a..6333250492 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config; import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test; class ImmutablePriorityPropertyDelegateTest { private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); @Test void getOverrideFromEnv() throws Exception { diff --git a/dolphinscheduler-common/src/test/resources/remote-logging.yaml b/dolphinscheduler-common/src/test/resources/remote-logging.yaml new file mode 100644 index 0000000000..cb149a77fe --- /dev/null +++ b/dolphinscheduler-common/src/test/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml index 9fc3a3b679..d521e53bc2 100644 --- a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml +++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml @@ -52,6 +52,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml index 70622942f0..5ac9b6350d 100644 --- a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml +++ b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf