From eab71f1071774bd51fe5cfeb66597ea988c49e7c Mon Sep 17 00:00:00 2001 From: calvin Date: Thu, 14 Mar 2024 17:57:13 +0800 Subject: [PATCH 01/88] [Improvement-15707][Master] Work out the issue that the workflow with a task dependency couldn't work well. (#15712) --- .../dao/mapper/ProcessInstanceMapper.java | 2 ++ .../dao/repository/ProcessInstanceDao.java | 4 ++- .../impl/ProcessInstanceDaoImpl.java | 4 ++- .../dao/mapper/ProcessInstanceMapper.xml | 26 +++++++++++++------ .../dao/mapper/ProcessInstanceMapperTest.java | 6 +++-- .../server/master/utils/DependentExecute.java | 11 +++++--- 6 files changed, 37 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java index deef2a9d4c..509bf7ca4d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java @@ -219,12 +219,14 @@ public interface ProcessInstanceMapper extends BaseMapper { * query last manual process instance * * @param definitionCode definitionCode + * @param taskCode taskCode * @param startTime startTime * @param endTime endTime * @param testFlag testFlag * @return process instance */ ProcessInstance queryLastManualProcess(@Param("processDefinitionCode") Long definitionCode, + @Param("taskCode") Long taskCode, @Param("startTime") Date startTime, @Param("endTime") Date endTime, @Param("testFlag") int testFlag); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ProcessInstanceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ProcessInstanceDao.java index 6aa48ea12d..c1bceab120 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ProcessInstanceDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ProcessInstanceDao.java @@ -51,10 +51,12 @@ public interface ProcessInstanceDao extends IDao { * find last manual process instance interval * * @param definitionCode process definition code + * @param taskCode taskCode * @param dateInterval dateInterval * @return process instance */ - ProcessInstance queryLastManualProcessInterval(Long definitionCode, DateInterval dateInterval, int testFlag); + ProcessInstance queryLastManualProcessInterval(Long definitionCode, Long taskCode, DateInterval dateInterval, + int testFlag); /** * query first schedule process instance diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImpl.java index fca93da29d..a5562f7a91 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImpl.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImpl.java @@ -82,13 +82,15 @@ public class ProcessInstanceDaoImpl extends BaseDao - + + + + + + + + + diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java index 17c1537184..13dcf91f55 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java @@ -27,7 +27,11 @@ import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue; import org.apache.dolphinscheduler.dao.repository.TaskGroupQueueDao; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.RandomUtils; + import java.util.Date; +import java.util.List; import org.assertj.core.util.Lists; import org.junit.jupiter.api.Test; @@ -55,6 +59,35 @@ class TaskGroupQueueDaoImplTest extends BaseDaoTest { assertEquals(1, taskGroupQueueDao.queryAllInQueueTaskGroupQueue().size()); } + @Test + void queryInQueueTaskGroupQueue_withMinId() { + // Insert 1w ~ 10w records + int insertCount = RandomUtils.nextInt(10000, 100000); + List insertTaskGroupQueue = Lists.newArrayList(); + for (int i = 0; i < insertCount; i++) { + TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS); + insertTaskGroupQueue.add(taskGroupQueue); + } + taskGroupQueueDao.insertBatch(insertTaskGroupQueue); + + int minTaskGroupQueueId = -1; + int limit = 1000; + int queryCount = 0; + while (true) { + List taskGroupQueues = + taskGroupQueueDao.queryInQueueTaskGroupQueue(minTaskGroupQueueId, limit); + if (CollectionUtils.isEmpty(taskGroupQueues)) { + break; + } + queryCount += taskGroupQueues.size(); + if (taskGroupQueues.size() < limit) { + break; + } + minTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId(); + } + assertEquals(insertCount, queryCount); + } + @Test void queryAllInQueueTaskGroupQueueByGroupId() { TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS); @@ -91,6 +124,49 @@ class TaskGroupQueueDaoImplTest extends BaseDaoTest { assertEquals(1, taskGroupQueueDao.queryAcquiredTaskGroupQueueByGroupId(1).size()); } + @Test + void countUsingTaskGroupQueueByGroupId() { + assertEquals(0, taskGroupQueueDao.countUsingTaskGroupQueueByGroupId(1)); + + TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS); + taskGroupQueueDao.insert(taskGroupQueue); + assertEquals(1, taskGroupQueueDao.countUsingTaskGroupQueueByGroupId(1)); + + taskGroupQueue = createTaskGroupQueue(Flag.YES, TaskGroupQueueStatus.WAIT_QUEUE); + taskGroupQueueDao.insert(taskGroupQueue); + assertEquals(1, taskGroupQueueDao.countUsingTaskGroupQueueByGroupId(1)); + } + + @Test + void queryWaitNotifyForceStartTaskGroupQueue() { + // Insert 1w records + int insertCount = RandomUtils.nextInt(10000, 20000); + List insertTaskGroupQueue = Lists.newArrayList(); + for (int i = 0; i < insertCount; i++) { + TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.YES, TaskGroupQueueStatus.ACQUIRE_SUCCESS); + + insertTaskGroupQueue.add(taskGroupQueue); + } + taskGroupQueueDao.insertBatch(insertTaskGroupQueue); + + int beginTaskGroupQueueId = -1; + int limit = 1000; + int queryCount = 0; + while (true) { + List taskGroupQueues = + taskGroupQueueDao.queryWaitNotifyForceStartTaskGroupQueue(beginTaskGroupQueueId, limit); + if (CollectionUtils.isEmpty(taskGroupQueues)) { + break; + } + queryCount += taskGroupQueues.size(); + if (taskGroupQueues.size() < limit) { + break; + } + beginTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId(); + } + assertEquals(insertCount, queryCount); + } + private TaskGroupQueue createTaskGroupQueue(Flag forceStart, TaskGroupQueueStatus taskGroupQueueStatus) { return TaskGroupQueue.builder() .taskId(1) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java index fae0d2b91f..bd7af94611 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java @@ -96,6 +96,8 @@ public class TaskGroupCoordinator extends BaseDaemonThread { @Autowired private ProcessInstanceDao processInstanceDao; + private static int DEFAULT_LIMIT = 1000; + public TaskGroupCoordinator() { super("TaskGroupCoordinator"); } @@ -147,10 +149,10 @@ public class TaskGroupCoordinator extends BaseDaemonThread { if (CollectionUtils.isEmpty(taskGroups)) { return; } + StopWatch taskGroupCoordinatorRoundTimeCost = StopWatch.createStarted(); + for (TaskGroup taskGroup : taskGroups) { - List taskGroupQueues = - taskGroupQueueDao.queryAcquiredTaskGroupQueueByGroupId(taskGroup.getId()); - int actualUseSize = taskGroupQueues.size(); + int actualUseSize = taskGroupQueueDao.countUsingTaskGroupQueueByGroupId(taskGroup.getId()); if (taskGroup.getUseSize() == actualUseSize) { continue; } @@ -160,13 +162,35 @@ public class TaskGroupCoordinator extends BaseDaemonThread { taskGroup.setUseSize(actualUseSize); taskGroupDao.updateById(taskGroup); } + log.info("Success amend TaskGroup useSize cost: {}/ms", taskGroupCoordinatorRoundTimeCost.getTime()); } /** * Make sure the TaskGroupQueue status is {@link TaskGroupQueueStatus#RELEASE} when the related {@link TaskInstance} is not exist or status is finished. */ private void amendTaskGroupQueueStatus() { - List taskGroupQueues = taskGroupQueueDao.queryAllInQueueTaskGroupQueue(); + int minTaskGroupQueueId = -1; + int limit = DEFAULT_LIMIT; + StopWatch taskGroupCoordinatorRoundTimeCost = StopWatch.createStarted(); + while (true) { + List taskGroupQueues = + taskGroupQueueDao.queryInQueueTaskGroupQueue(minTaskGroupQueueId, limit); + if (CollectionUtils.isEmpty(taskGroupQueues)) { + break; + } + amendTaskGroupQueueStatus(taskGroupQueues); + if (taskGroupQueues.size() < limit) { + break; + } + minTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId(); + } + log.info("Success amend TaskGroupQueue status cost: {}/ms", taskGroupCoordinatorRoundTimeCost.getTime()); + } + + /** + * Make sure the TaskGroupQueue status is {@link TaskGroupQueueStatus#RELEASE} when the related {@link TaskInstance} is not exist or status is finished. + */ + private void amendTaskGroupQueueStatus(List taskGroupQueues) { List taskInstanceIds = taskGroupQueues.stream() .map(TaskGroupQueue::getTaskId) .collect(Collectors.toList()); @@ -198,10 +222,30 @@ public class TaskGroupCoordinator extends BaseDaemonThread { // Find the force start task group queue(Which is inQueue and forceStart is YES) // Notify the related waiting task instance // Set the taskGroupQueue status to RELEASE and remove it from queue - List taskGroupQueues = taskGroupQueueDao.queryAllInQueueTaskGroupQueue() - .stream() - .filter(taskGroupQueue -> Flag.YES.getCode() == taskGroupQueue.getForceStart()) - .collect(Collectors.toList()); + // We use limit here to avoid OOM, and we will retry to notify force start queue at next time + int minTaskGroupQueueId = -1; + int limit = DEFAULT_LIMIT; + StopWatch taskGroupCoordinatorRoundTimeCost = StopWatch.createStarted(); + while (true) { + List taskGroupQueues = + taskGroupQueueDao.queryWaitNotifyForceStartTaskGroupQueue(minTaskGroupQueueId, limit); + if (CollectionUtils.isEmpty(taskGroupQueues)) { + break; + } + dealWithForceStartTaskGroupQueue(taskGroupQueues); + if (taskGroupQueues.size() < limit) { + break; + } + minTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId(); + } + log.info("Success deal with force start TaskGroupQueue cost: {}/ms", + taskGroupCoordinatorRoundTimeCost.getTime()); + } + + private void dealWithForceStartTaskGroupQueue(List taskGroupQueues) { + // Find the force start task group queue(Which is inQueue and forceStart is YES) + // Notify the related waiting task instance + // Set the taskGroupQueue status to RELEASE and remove it from queue for (TaskGroupQueue taskGroupQueue : taskGroupQueues) { try { LogUtils.setTaskInstanceIdMDC(taskGroupQueue.getTaskId()); From d39bdcb165c069792fd33cad5c1fb441f7c5ae47 Mon Sep 17 00:00:00 2001 From: John Huang Date: Sun, 31 Mar 2024 21:47:26 +0800 Subject: [PATCH 26/88] [RemoteLogging] Move init into loghandler (#15780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 旺阳 --- .../log/remote/GcsRemoteLogHandler.java | 21 +++++++------------ .../log/remote/OssRemoteLogHandler.java | 21 +++++++------------ .../common/log/remote/S3RemoteLogHandler.java | 21 +++++++------------ 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/GcsRemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/GcsRemoteLogHandler.java index 20fd30336e..ad6e534251 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/GcsRemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/GcsRemoteLogHandler.java @@ -49,19 +49,6 @@ public class GcsRemoteLogHandler implements RemoteLogHandler, Closeable { private static GcsRemoteLogHandler instance; private GcsRemoteLogHandler() { - - } - - public static synchronized GcsRemoteLogHandler getInstance() { - if (instance == null) { - instance = new GcsRemoteLogHandler(); - instance.init(); - } - - return instance; - } - - public void init() { try { credential = readCredentials(); bucketName = readBucketName(); @@ -73,6 +60,14 @@ public class GcsRemoteLogHandler implements RemoteLogHandler, Closeable { } } + public static synchronized GcsRemoteLogHandler getInstance() { + if (instance == null) { + instance = new GcsRemoteLogHandler(); + } + + return instance; + } + @Override public void close() throws IOException { try { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/OssRemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/OssRemoteLogHandler.java index 792085b194..59b1394515 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/OssRemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/OssRemoteLogHandler.java @@ -44,19 +44,6 @@ public class OssRemoteLogHandler implements RemoteLogHandler, Closeable { private static OssRemoteLogHandler instance; private OssRemoteLogHandler() { - - } - - public static synchronized OssRemoteLogHandler getInstance() { - if (instance == null) { - instance = new OssRemoteLogHandler(); - instance.init(); - } - - return instance; - } - - public void init() { String accessKeyId = readOssAccessKeyId(); String accessKeySecret = readOssAccessKeySecret(); String endpoint = readOssEndpoint(); @@ -66,6 +53,14 @@ public class OssRemoteLogHandler implements RemoteLogHandler, Closeable { checkBucketNameExists(bucketName); } + public static synchronized OssRemoteLogHandler getInstance() { + if (instance == null) { + instance = new OssRemoteLogHandler(); + } + + return instance; + } + @Override public void sendRemoteLog(String logPath) { String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java index d1c8c41445..54dba2d5bd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java @@ -56,19 +56,6 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { private static S3RemoteLogHandler instance; private S3RemoteLogHandler() { - - } - - public static synchronized S3RemoteLogHandler getInstance() { - if (instance == null) { - instance = new S3RemoteLogHandler(); - instance.init(); - } - - return instance; - } - - public void init() { accessKeyId = readAccessKeyID(); accessKeySecret = readAccessKeySecret(); region = readRegion(); @@ -78,6 +65,14 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { checkBucketNameExists(bucketName); } + public static synchronized S3RemoteLogHandler getInstance() { + if (instance == null) { + instance = new S3RemoteLogHandler(); + } + + return instance; + } + protected AmazonS3 buildS3Client() { if (StringUtils.isNotEmpty(endPoint)) { return AmazonS3ClientBuilder From ac0189a636ea91715f2fbaf1e79e8e9e27b07bee Mon Sep 17 00:00:00 2001 From: John Huang Date: Mon, 1 Apr 2024 09:50:49 +0800 Subject: [PATCH 27/88] [DSIP-24][RemoteLogging]Support AbsRemoteLogHandler (#15769) --- docs/docs/en/guide/remote-logging.md | 12 +- docs/docs/zh/guide/remote-logging.md | 12 +- dolphinscheduler-common/pom.xml | 5 + .../common/constants/Constants.java | 7 + .../log/remote/AbsRemoteLogHandler.java | 137 +++++++++++++++++ .../log/remote/RemoteLogHandlerFactory.java | 2 + .../src/main/resources/common.properties | 7 +- .../log/remote/AbsRemoteLogHandlerTest.java | 144 ++++++++++++++++++ 8 files changed, 313 insertions(+), 13 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandler.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandlerTest.java diff --git a/docs/docs/en/guide/remote-logging.md b/docs/docs/en/guide/remote-logging.md index a29dc06582..7753fe4116 100644 --- a/docs/docs/en/guide/remote-logging.md +++ b/docs/docs/en/guide/remote-logging.md @@ -10,7 +10,7 @@ If you deploy DolphinScheduler in `Standalone` mode, you only need to configure ```properties # Whether to enable remote logging remote.logging.enable=false -# if remote.logging.enable = true, set the target of remote logging +# if remote.logging.enable = true, set the target of remote logging, currently support OSS, S3, GCS, ABS remote.logging.target=OSS # if remote.logging.enable = true, set the log base directory remote.logging.base.dir=logs @@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name= Configure `common.properties` as follows: ```properties -# abs container name, required if you set resource.storage.type=ABS -resource.azure.blob.storage.container.name= # abs account name, required if you set resource.storage.type=ABS -resource.azure.blob.storage.account.name= -# abs connection string, required if you set resource.storage.type=ABS -resource.azure.blob.storage.connection.string= +remote.logging.abs.account.name= +# abs account key, required if you set resource.storage.type=ABS +remote.logging.abs.account.key= +# abs container name, required if you set resource.storage.type=ABS +remote.logging.abs.container.name= ``` ### Notice diff --git a/docs/docs/zh/guide/remote-logging.md b/docs/docs/zh/guide/remote-logging.md index 7321badb1a..0e45353636 100644 --- a/docs/docs/zh/guide/remote-logging.md +++ b/docs/docs/zh/guide/remote-logging.md @@ -10,7 +10,7 @@ Apache DolphinScheduler支持将任务日志传输到远端存储上。当配置 ```properties # 是否开启远程日志存储 remote.logging.enable=true -# 任务日志写入的远端存储，目前支持OSS, S3, GCS +# 任务日志写入的远端存储，目前支持OSS, S3, GCS, ABS remote.logging.target=OSS # 任务日志在远端存储上的目录 remote.logging.base.dir=logs @@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name= 配置`common.propertis`如下： ```properties -# abs container name, required if you set resource.storage.type=ABS -resource.azure.blob.storage.container.name= # abs account name, required if you set resource.storage.type=ABS -resource.azure.blob.storage.account.name= -# abs connection string, required if you set resource.storage.type=ABS -resource.azure.blob.storage.connection.string= +remote.logging.abs.account.name= +# abs account key, required if you set resource.storage.type=ABS +remote.logging.abs.account.key= +# abs container name, required if you set resource.storage.type=ABS +remote.logging.abs.container.name= ``` ### 注意事项 diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index b03fdd7483..eda9a72e30 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -98,6 +98,11 @@ esdk-obs-java-bundle + + com.azure + azure-storage-blob + + com.github.oshi oshi-core diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 7556636216..054a9410d5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -725,6 +725,13 @@ public final class Constants { public static final String REMOTE_LOGGING_GCS_BUCKET_NAME = "remote.logging.google.cloud.storage.bucket.name"; + /** + * remote logging for ABS + */ + public static final String REMOTE_LOGGING_ABS_ACCOUNT_NAME = "remote.logging.abs.account.name"; + public static final String REMOTE_LOGGING_ABS_ACCOUNT_KEY = "remote.logging.abs.account.key"; + public static final String REMOTE_LOGGING_ABS_CONTAINER_NAME = "remote.logging.abs.container.name"; + /** * data quality */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandler.java new file mode 100644 index 0000000000..c0df3f6287 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandler.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.log.remote; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.io.Closeable; +import java.io.FileOutputStream; +import java.io.IOException; + +import lombok.extern.slf4j.Slf4j; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.common.StorageSharedKeyCredential; + +@Slf4j +public class AbsRemoteLogHandler implements RemoteLogHandler, Closeable { + + private String accountName; + + private String accountKey; + + private String containerName; + + private BlobContainerClient blobContainerClient; + + private static AbsRemoteLogHandler instance; + + private AbsRemoteLogHandler() { + accountName = readAccountName(); + accountKey = readAccountKey(); + containerName = readContainerName(); + blobContainerClient = buildBlobContainerClient(); + } + + public static synchronized AbsRemoteLogHandler getInstance() { + if (instance == null) { + instance = new AbsRemoteLogHandler(); + } + + return instance; + } + + protected BlobContainerClient buildBlobContainerClient() { + + BlobServiceClient serviceClient = new BlobServiceClientBuilder() + .endpoint(String.format("https://%s.blob.core.windows.net/", accountName)) + .credential(new StorageSharedKeyCredential(accountName, accountKey)) + .buildClient(); + + if (StringUtils.isBlank(containerName)) { + throw new IllegalArgumentException("remote.logging.abs.container.name is blank"); + } + + try { + this.blobContainerClient = serviceClient.getBlobContainerClient(containerName); + } catch (Exception ex) { + throw new IllegalArgumentException( + "containerName: " + containerName + " is not exists, you need to create them by yourself"); + } + + log.info("containerName: {} has been found.", containerName); + + return blobContainerClient; + } + + @Override + public void close() throws IOException { + // no need to close blobContainerClient + } + + @Override + public void sendRemoteLog(String logPath) { + String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); + + try { + log.info("send remote log {} to Azure Blob {}", logPath, objectName); + blobContainerClient.getBlobClient(objectName).uploadFromFile(logPath); + } catch (Exception e) { + log.error("error while sending remote log {} to Azure Blob {}", logPath, objectName, e); + } + } + + @Override + public void getRemoteLog(String logPath) { + String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); + + try { + log.info("get remote log on Azure Blob {} to {}", objectName, logPath); + + try ( + BlobInputStream bis = blobContainerClient.getBlobClient(objectName).openInputStream(); + FileOutputStream fos = new FileOutputStream(logPath)) { + byte[] readBuf = new byte[1024]; + int readLen = 0; + while ((readLen = bis.read(readBuf)) > 0) { + fos.write(readBuf, 0, readLen); + } + } + } catch (Exception e) { + log.error("error while getting remote log on Azure Blob {} to {}", objectName, logPath, e); + } + } + + protected String readAccountName() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_NAME); + } + + protected String readAccountKey() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_KEY); + } + + protected String readContainerName() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_CONTAINER_NAME); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java index 73ab41a134..ac75a23f2d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java @@ -39,6 +39,8 @@ public class RemoteLogHandlerFactory { return S3RemoteLogHandler.getInstance(); } else if ("GCS".equals(target)) { return GcsRemoteLogHandler.getInstance(); + } else if ("ABS".equals(target)) { + return AbsRemoteLogHandler.getInstance(); } log.error("No suitable remote logging target for {}", target); diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 669d3dfef3..fdb553b4bc 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -202,4 +202,9 @@ remote.logging.s3.region= remote.logging.google.cloud.storage.credential=/path/to/credential # gcs bucket name, required if you set remote.logging.target=GCS remote.logging.google.cloud.storage.bucket.name= - +# abs account name, required if you set resource.storage.type=ABS +remote.logging.abs.account.name= +# abs account key, required if you set resource.storage.type=ABS +remote.logging.abs.account.key= +# abs container name, required if you set resource.storage.type=ABS +remote.logging.abs.container.name= diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandlerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandlerTest.java new file mode 100644 index 0000000000..bc18f952a9 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/remote/AbsRemoteLogHandlerTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.log.remote; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.LogUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.common.StorageSharedKeyCredential; + +@Slf4j +@ExtendWith(MockitoExtension.class) +public class AbsRemoteLogHandlerTest { + + @Mock + BlobServiceClient blobServiceClient; + + @Mock + BlobContainerClient blobContainerClient; + + @Mock + BlobClient blobClient; + + @Test + public void testAbsRemoteLogHandlerContainerNameBlack() { + try ( + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + MockedStatic remoteLogUtilsMockedStatic = Mockito.mockStatic(LogUtils.class)) { + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_NAME)) + .thenReturn("account_name"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_KEY)) + .thenReturn("account_key"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_CONTAINER_NAME)) + .thenReturn(""); + remoteLogUtilsMockedStatic.when(LogUtils::getLocalLogBaseDir).thenReturn("logs"); + + IllegalArgumentException thrown = Assertions.assertThrows(IllegalArgumentException.class, () -> { + AbsRemoteLogHandler.getInstance(); + }); + Assertions.assertEquals("remote.logging.abs.container.name is blank", thrown.getMessage()); + } + } + + @Test + public void testAbsRemoteLogHandlerContainerNotExists() { + try ( + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + MockedStatic remoteLogUtilsMockedStatic = Mockito.mockStatic(LogUtils.class); + MockedConstruction k8sClientWrapperMockedConstruction = + Mockito.mockConstruction(BlobServiceClientBuilder.class, (mock, context) -> { + when(mock.endpoint(any(String.class))).thenReturn(mock); + when(mock.credential(any(StorageSharedKeyCredential.class))).thenReturn(mock); + when(mock.buildClient()) + .thenReturn(blobServiceClient); + })) { + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_NAME)) + .thenReturn("account_name"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_KEY)) + .thenReturn("account_key"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_CONTAINER_NAME)) + .thenReturn("container_name"); + remoteLogUtilsMockedStatic.when(LogUtils::getLocalLogBaseDir).thenReturn("logs"); + + when(blobServiceClient.getBlobContainerClient(any(String.class))).thenThrow( + new NullPointerException("container not exists")); + IllegalArgumentException thrown = Assertions.assertThrows(IllegalArgumentException.class, () -> { + AbsRemoteLogHandler.getInstance(); + }); + Assertions.assertEquals("containerName: container_name is not exists, you need to create them by yourself", + thrown.getMessage()); + } + } + + @Test + public void testAbsRemoteLogHandler() { + + try ( + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + MockedStatic remoteLogUtilsMockedStatic = Mockito.mockStatic(LogUtils.class); + MockedConstruction blobServiceClientBuilderMockedConstruction = + Mockito.mockConstruction(BlobServiceClientBuilder.class, (mock, context) -> { + when(mock.endpoint(any(String.class))).thenReturn(mock); + when(mock.credential(any(StorageSharedKeyCredential.class))).thenReturn(mock); + when(mock.buildClient()) + .thenReturn(blobServiceClient); + }); + MockedStatic remoteLogUtilsMockedStatic1 = Mockito.mockStatic(RemoteLogUtils.class)) { + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_NAME)) + .thenReturn("account_name"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_ACCOUNT_KEY)) + .thenReturn("account_key"); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_ABS_CONTAINER_NAME)) + .thenReturn("container_name"); + remoteLogUtilsMockedStatic.when(LogUtils::getLocalLogBaseDir).thenReturn("logs"); + String logPath = "logpath"; + String objectName = "objectname"; + remoteLogUtilsMockedStatic1.when(() -> RemoteLogUtils.getObjectNameFromLogPath(logPath)) + .thenReturn(objectName); + + when(blobServiceClient.getBlobContainerClient(any(String.class))).thenReturn(blobContainerClient); + when(blobContainerClient.getBlobClient(objectName)).thenReturn(blobClient); + + AbsRemoteLogHandler absRemoteLogHandler = AbsRemoteLogHandler.getInstance(); + Assertions.assertNotNull(absRemoteLogHandler); + + absRemoteLogHandler.sendRemoteLog(logPath); + Mockito.verify(blobClient, times(1)).uploadFromFile(logPath); + } + } +} From 8acc69794275149a359a711359873b39781f58b7 Mon Sep 17 00:00:00 2001 From: caishunfeng Date: Mon, 1 Apr 2024 22:49:59 +0800 Subject: [PATCH 28/88] [Improvement] add resource full name check (#15786) * [Improvement] add resource full name check --- .../dolphinscheduler/api/enums/Status.java | 2 + .../api/service/ResourcesService.java | 6 +- .../service/impl/ResourcesServiceImpl.java | 20 +- .../api/service/ResourcesServiceTest.java | 329 ++++++++---------- 4 files changed, 166 insertions(+), 191 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 113ccb6bd1..9dc12d9ba0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -323,6 +323,8 @@ public enum Status { REMOVE_TASK_INSTANCE_CACHE_ERROR(20019, "remove task instance cache error", "删除任务实例缓存错误"), + ILLEGAL_RESOURCE_PATH(20020, "Resource file [{0}] is illegal", "非法的资源路径[{0}]"), + USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission", "当前用户[{0}]没有[{1}]项目的操作权限"), USER_NO_WRITE_PROJECT_PERM(30003, "user [{0}] does not have write permission for project [{1}]", diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 24d1ba8727..54023baad9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -194,13 +194,13 @@ public interface ResourcesService { org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException; /** - * Get resource by given resource type and full name. + * Get resource by given resource type and file name. * Useful in Python API create task which need processDefinition information. * * @param userName user who query resource - * @param fullName full name of the resource + * @param fileName file name of the resource */ - StorageEntity queryFileStatus(String userName, String fullName) throws Exception; + StorageEntity queryFileStatus(String userName, String fileName) throws Exception; /** * delete DATA_TRANSFER data in resource center diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index e6341f021d..6a15da17a8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -126,6 +126,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } String tenantCode = getTenantCode(user); + checkFullName(tenantCode, currentDir); String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode) : storageOperate.getResDir(tenantCode); @@ -171,6 +172,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } String tenantCode = getTenantCode(user); + checkFullName(tenantCode, currentDir); result = verifyFile(name, type, file); if (!result.getCode().equals(Status.SUCCESS.getCode())) { @@ -257,6 +259,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } String tenantCode = getTenantCode(user); + checkFullName(tenantCode, resourceFullName); if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { log.error("current user does not have permission"); @@ -264,7 +267,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - String defaultPath = storageOperate.getResDir(tenantCode); + String defaultPath = storageOperate.getDir(type, tenantCode); StorageEntity resource; try { @@ -949,6 +952,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } String tenantCode = getTenantCode(user); + checkFullName(tenantCode, currentDir); if (FileUtils.directoryTraversal(fileName)) { log.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName)); @@ -1280,9 +1284,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } private void checkFullName(String userTenantCode, String fullName) { + if (StringUtils.isEmpty(fullName)) { + return; + } + if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { + return; + } + // Avoid returning to the parent directory + if (fullName.contains("../")) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); + } String baseDir = storageOperate.getDir(ResourceType.ALL, userTenantCode); - if (StringUtils.isNotBlank(fullName) && !StringUtils.startsWith(fullName, baseDir)) { - throw new ServiceException("Resource file: " + fullName + " is illegal"); + if (!StringUtils.startsWith(fullName, baseDir)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); } } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 0679b8892d..6e94a25861 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.service; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -70,8 +69,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockMultipartFile; import com.google.common.io.Files; @@ -83,9 +80,10 @@ import com.google.common.io.Files; @MockitoSettings(strictness = Strictness.LENIENT) public class ResourcesServiceTest { - private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); - + private static final String basePath = "/dolphinscheduler"; private static final String tenantCode = "123"; + private static final String tenantFileResourceDir = "/dolphinscheduler/123/resources/"; + private static final String tenantUdfResourceDir = "/dolphinscheduler/123/udfs/"; @InjectMocks private ResourcesServiceImpl resourcesService; @@ -153,18 +151,30 @@ public class ResourcesServiceTest { // CURRENT_LOGIN_USER_TENANT_NOT_EXIST when(userMapper.selectById(user.getId())).thenReturn(getUser()); when(tenantMapper.queryById(1)).thenReturn(null); - Assertions.assertThrows(ServiceException.class, + ServiceException serviceException = Assertions.assertThrows(ServiceException.class, () -> resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()), "/")); + assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), serviceException.getMessage()); + // set tenant for user user.setTenantId(1); when(tenantMapper.queryById(1)).thenReturn(getTenant()); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + + // ILLEGAL_RESOURCE_FILE + String illegal_path = "/dolphinscheduler/123/../"; + serviceException = Assertions.assertThrows(ServiceException.class, + () -> { + MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); + resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, + mockMultipartFile, illegal_path); + }); + assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, illegal_path), serviceException); // RESOURCE_FILE_IS_EMPTY MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); Result result = resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, - mockMultipartFile, "/"); - logger.info(result.toString()); + mockMultipartFile, tenantFileResourceDir); assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg()); // RESOURCE_SUFFIX_FORBID_CHANGE @@ -172,8 +182,7 @@ public class ResourcesServiceTest { when(Files.getFileExtension("test.pdf")).thenReturn("pdf"); when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); result = resourcesService.uploadResource(user, "ResourcesServiceTest.jar", ResourceType.FILE, mockMultipartFile, - "/"); - logger.info(result.toString()); + tenantFileResourceDir); assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg()); // UDF_RESOURCE_SUFFIX_NOT_JAR @@ -181,45 +190,42 @@ public class ResourcesServiceTest { new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes()); when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf"); result = resourcesService.uploadResource(user, "ResourcesServiceTest.pdf", ResourceType.UDF, mockMultipartFile, - "/"); - logger.info(result.toString()); + tenantUdfResourceDir); assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg()); // FULL_FILE_NAME_TOO_LONG String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf"; mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes()); when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf"); + // '/databasePath/tenantCode/RESOURCE/' - when(storageOperate.getResDir(tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); - result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile, "/"); - logger.info(result.toString()); + when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); + result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile, + tenantFileResourceDir); assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg()); } @Test - public void testCreateDirecotry() { + public void testCreateDirecotry() throws IOException { User user = new User(); user.setId(1); user.setUserType(UserType.GENERAL_USER); + String fileName = "directoryTest"; // RESOURCE_EXIST user.setId(1); user.setTenantId(1); when(tenantMapper.queryById(1)).thenReturn(getTenant()); when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(storageOperate.getResDir(tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); - try { - when(storageOperate.exists("/dolphinscheduler/123/resources/directoryTest")).thenReturn(true); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - Result result = resourcesService.createDirectory(user, "directoryTest", ResourceType.FILE, -1, "/"); - logger.info(result.toString()); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true); + Result result = resourcesService.createDirectory(user, fileName, ResourceType.FILE, -1, tenantFileResourceDir); assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); } @Test - public void testUpdateResource() { + public void testUpdateResource() throws Exception { User user = new User(); user.setId(1); user.setUserType(UserType.GENERAL_USER); @@ -227,7 +233,13 @@ public class ResourcesServiceTest { when(userMapper.selectById(user.getId())).thenReturn(getUser()); when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(storageOperate.getResDir(tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); + + // TENANT_NOT_EXIST + when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); + Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user, + "ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null)); // USER_NO_OPERATION_PERM user.setUserType(UserType.GENERAL_USER); @@ -235,92 +247,58 @@ public class ResourcesServiceTest { Tenant tenantWNoPermission = new Tenant(); tenantWNoPermission.setTenantCode("321"); when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission); - Result result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest", - tenantCode, "ResourcesServiceTest", ResourceType.FILE, null); - logger.info(result.toString()); + when(storageOperate.getDir(ResourceType.ALL, "321")).thenReturn(basePath); + + String fileName = "ResourcesServiceTest"; + Result result = resourcesService.updateResource(user, tenantFileResourceDir + fileName, + tenantCode, fileName, ResourceType.FILE, null); assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg()); // SUCCESS when(tenantMapper.queryById(1)).thenReturn(getTenant()); - try { - when(storageOperate.exists(Mockito.any())).thenReturn(false); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } + when(storageOperate.exists(Mockito.any())).thenReturn(false); - try { - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest", - "/dolphinscheduler/123/resources/", tenantCode, ResourceType.FILE)) - .thenReturn(getStorageEntityResource()); - result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest", - tenantCode, "ResourcesServiceTest", ResourceType.FILE, null); - logger.info(result.toString()); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } catch (Exception e) { - logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest", - e); - } + when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, + tenantFileResourceDir, tenantCode, ResourceType.FILE)) + .thenReturn(getStorageEntityResource(fileName)); + result = resourcesService.updateResource(user, tenantFileResourceDir + fileName, + tenantCode, fileName, ResourceType.FILE, null); + assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); // Tests for udf resources. - // RESOURCE_EXIST - try { - when(storageOperate.exists("/dolphinscheduler/123/resources/ResourcesServiceTest2.jar")).thenReturn(true); - } catch (IOException e) { - logger.error("error occurred when checking resource: " - + "/dolphinscheduler/123/resources/ResourcesServiceTest2.jar"); - } - - try { - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", - "/dolphinscheduler/123/resources/", tenantCode, ResourceType.UDF)) - .thenReturn(getStorageEntityUdfResource()); - } catch (Exception e) { - logger.error(e.getMessage() + " Resource path: {}", - "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", e); - } - result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", - tenantCode, "ResourcesServiceTest2.jar", ResourceType.UDF, null); - logger.info(result.toString()); - assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); - - // TENANT_NOT_EXIST - when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user, - "ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null)); - - // SUCCESS - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - - result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", - tenantCode, "ResourcesServiceTest1.jar", ResourceType.UDF, null); - logger.info(result.toString()); + fileName = "ResourcesServiceTest.jar"; + when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir); + when(storageOperate.exists(tenantUdfResourceDir + fileName)).thenReturn(true); + when(storageOperate.getFileStatus(tenantUdfResourceDir + fileName, tenantUdfResourceDir, tenantCode, + ResourceType.UDF)) + .thenReturn(getStorageEntityUdfResource(fileName)); + result = resourcesService.updateResource(user, tenantUdfResourceDir + fileName, + tenantCode, fileName, ResourceType.UDF, null); assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testQueryResourceListPaging() { + public void testQueryResourceListPaging() throws Exception { User loginUser = new User(); loginUser.setId(1); loginUser.setTenantId(1); loginUser.setTenantCode("tenant1"); loginUser.setUserType(UserType.ADMIN_USER); - List mockResList = new ArrayList(); - mockResList.add(getStorageEntityResource()); - List mockUserList = new ArrayList(); + + String fileName = "ResourcesServiceTest"; + List mockResList = new ArrayList<>(); + mockResList.add(getStorageEntityResource(fileName)); + List mockUserList = new ArrayList<>(); mockUserList.add(getUser()); when(userMapper.selectList(null)).thenReturn(mockUserList); when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); - when(storageOperate.getResDir(tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); + when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.listFilesStatus(tenantFileResourceDir, tenantFileResourceDir, + tenantCode, ResourceType.FILE)).thenReturn(mockResList); - try { - when(storageOperate.listFilesStatus("/dolphinscheduler/123/resources/", "/dolphinscheduler/123/resources/", - tenantCode, ResourceType.FILE)).thenReturn(mockResList); - } catch (Exception e) { - logger.error("QueryResourceListPaging Error"); - } Result result = resourcesService.queryResourceListPaging(loginUser, "", "", ResourceType.FILE, "Test", 1, 10); - logger.info(result.toString()); assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); PageInfo pageInfo = (PageInfo) result.getData(); Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); @@ -330,29 +308,30 @@ public class ResourcesServiceTest { @Test public void testQueryResourceList() { User loginUser = getUser(); + String fileName = "ResourcesServiceTest"; when(userMapper.selectList(null)).thenReturn(Collections.singletonList(loginUser)); when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser); when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn("/dolphinscheduler"); - when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); - when(storageOperate.getResDir(tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); - when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/resources/", - "/dolphinscheduler/123/resources/", tenantCode, ResourceType.FILE)) - .thenReturn(Collections.singletonList(getStorageEntityResource())); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.listFilesStatusRecursively(tenantFileResourceDir, + tenantFileResourceDir, tenantCode, ResourceType.FILE)) + .thenReturn(Collections.singletonList(getStorageEntityResource(fileName))); Map result = - resourcesService.queryResourceList(loginUser, ResourceType.FILE, "/dolphinscheduler/123/resources/"); + resourcesService.queryResourceList(loginUser, ResourceType.FILE, tenantFileResourceDir); assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resourceList = (List) result.get(Constants.DATA_LIST); Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); // test udf - when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn("/dolphinscheduler/123/udfs/"); - when(storageOperate.getUdfDir(tenantCode)).thenReturn("/dolphinscheduler/123/udfs/"); - when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/udfs/", "/dolphinscheduler/123/udfs/", - tenantCode, ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource())); + when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir); + when(storageOperate.getUdfDir(tenantCode)).thenReturn(tenantUdfResourceDir); + when(storageOperate.listFilesStatusRecursively(tenantUdfResourceDir, tenantUdfResourceDir, + tenantCode, ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource("test.jar"))); loginUser.setUserType(UserType.GENERAL_USER); - result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, "/dolphinscheduler/123/udfs/"); + result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, tenantUdfResourceDir); assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); resourceList = (List) result.get(Constants.DATA_LIST); Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); @@ -360,7 +339,6 @@ public class ResourcesServiceTest { @Test public void testDelete() throws Exception { - User loginUser = new User(); loginUser.setId(0); loginUser.setUserType(UserType.GENERAL_USER); @@ -372,46 +350,40 @@ public class ResourcesServiceTest { Assertions.assertThrows(ServiceException.class, () -> resourcesService.delete(loginUser, "", "")); // RESOURCE_NOT_EXIST + String fileName = "ResourcesServiceTest"; when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn("/dolphinscheduler"); - when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn("/dolphinscheduler/123/resources/"); - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest", - "/dolphinscheduler/123/resources/", tenantCode, null)) - .thenReturn(getStorageEntityResource()); - Result result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResNotExist", tenantCode); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir); + when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, tenantFileResourceDir, tenantCode, null)) + .thenReturn(getStorageEntityResource(fileName)); + Result result = resourcesService.delete(loginUser, tenantFileResourceDir + "ResNotExist", tenantCode); assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // SUCCESS loginUser.setTenantId(1); - result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResourcesServiceTest", tenantCode); + result = resourcesService.delete(loginUser, tenantFileResourceDir + fileName, tenantCode); assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testVerifyResourceName() { - + public void testVerifyResourceName() throws IOException { User user = new User(); user.setId(1); user.setUserType(UserType.GENERAL_USER); - try { - when(storageOperate.exists("/ResourcesServiceTest.jar")).thenReturn(true); - } catch (IOException e) { - logger.error("error occurred when checking resource: /ResourcesServiceTest.jar\""); - } - Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); - logger.info(result.toString()); + + String fileName = "ResourcesServiceTest"; + when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true); + + Result result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user); assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); // RESOURCE_FILE_EXIST - result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); - logger.info(result.toString()); + result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user); Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode()); // SUCCESS result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user); - logger.info(result.toString()); assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } @Test @@ -441,8 +413,8 @@ public class ResourcesServiceTest { // SUCCESS when(FileUtils.getResourceViewSuffixes()).thenReturn("jar,sh"); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn("/dolphinscheduler"); - when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn("/dolphinscheduler/123/resources/"); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir); when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); when(storageOperate.exists(Mockito.any())).thenReturn(true); @@ -465,15 +437,16 @@ public class ResourcesServiceTest { exception.getMessage().contains("Not allow create or update resources without extension name")); // SUCCESS - when(storageOperate.getResDir(user.getTenantCode())).thenReturn("/dolphinscheduler/123/resources/"); + String fileName = "ResourcesServiceTest"; + when(storageOperate.getResDir(user.getTenantCode())).thenReturn(tenantFileResourceDir); when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any())) - .thenReturn(getStorageEntityResource()); + .thenReturn(getStorageEntityResource(fileName)); StorageEntity storageEntity = resourcesService.createOrUpdateResource(user.getUserName(), "filename.txt", "my-content"); Assertions.assertNotNull(storageEntity); - assertEquals("/dolphinscheduler/123/resources/ResourcesServiceTest", storageEntity.getFullName()); + assertEquals(tenantFileResourceDir + fileName, storageEntity.getFullName()); } @Test @@ -482,33 +455,35 @@ public class ResourcesServiceTest { when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); when(tenantMapper.queryById(1)).thenReturn(getTenant()); when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/tmp"); + + String fileName = "ResourcesServiceTest.jar"; ServiceException serviceException = Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(), - "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", tenantCode, "content")); - assertTrue(serviceException.getMessage() - .contains("Resource file: /dolphinscheduler/123/resources/ResourcesServiceTest.jar is illegal")); + tenantFileResourceDir + fileName, tenantCode, "content")); + assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, tenantFileResourceDir + fileName), + serviceException); // RESOURCE_NOT_EXIST - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn("/dolphinscheduler"); - when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/dolphinscheduler/123/resources"); - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "", tenantCode, - ResourceType.FILE)).thenReturn(null); - Result result = resourcesService.updateResourceContent(getUser(), - "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", tenantCode, "content"); + when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); + when(storageOperate.getResDir(Mockito.anyString())).thenReturn(tenantFileResourceDir); + when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, ResourceType.FILE)) + .thenReturn(null); + Result result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + fileName, tenantCode, + "content"); assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources", "", tenantCode, ResourceType.FILE)) - .thenReturn(getStorageEntityResource()); + when(storageOperate.getFileStatus(tenantFileResourceDir, "", tenantCode, ResourceType.FILE)) + .thenReturn(getStorageEntityResource(fileName)); - result = resourcesService.updateResourceContent(getUser(), "/dolphinscheduler/123/resources", tenantCode, + result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir, tenantCode, "content"); assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); // USER_NOT_EXIST when(userMapper.selectById(getUser().getId())).thenReturn(null); - result = resourcesService.updateResourceContent(getUser(), "/dolphinscheduler/123/resources/123.class", + result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + "123.class", tenantCode, "content"); Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); @@ -517,11 +492,11 @@ public class ResourcesServiceTest { when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); when(tenantMapper.queryById(1)).thenReturn(null); Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(), - "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", tenantCode, "content")); + tenantFileResourceDir + fileName, tenantCode, "content")); // SUCCESS - when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "", tenantCode, - ResourceType.FILE)).thenReturn(getStorageEntityResource()); + when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, + ResourceType.FILE)).thenReturn(getStorageEntityResource(fileName)); when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar"); when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); @@ -530,32 +505,25 @@ public class ResourcesServiceTest { when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); result = resourcesService.updateResourceContent(getUser(), - "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", tenantCode, "content"); - logger.info(result.toString()); + tenantFileResourceDir + fileName, tenantCode, "content"); assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testDownloadResource() { + public void testDownloadResource() throws IOException { when(tenantMapper.queryById(1)).thenReturn(getTenant()); when(userMapper.selectById(1)).thenReturn(getUser()); org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class); Path path = Mockito.mock(Path.class); when(Paths.get(Mockito.any())).thenReturn(path); - try { - when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L); - // resource null - org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), ""); - Assertions.assertNull(resource); - - when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock); - resource = resourcesService.downloadResource(getUser(), ""); - Assertions.assertNotNull(resource); - } catch (Exception e) { - logger.error("DownloadResource error", e); - Assertions.assertTrue(false); - } + when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L); + // resource null + org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), ""); + Assertions.assertNull(resource); + when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock); + resource = resourcesService.downloadResource(getUser(), ""); + Assertions.assertNotNull(resource); } @Test @@ -605,30 +573,21 @@ public class ResourcesServiceTest { } @Test - public void testCatFile() { + public void testCatFile() throws IOException { // SUCCESS - try { - List list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10)); - Assertions.assertNotNull(list); - - } catch (IOException e) { - logger.error("hadoop error", e); - } + List list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10)); + Assertions.assertNotNull(list); } @Test - void testQueryBaseDir() { + void testQueryBaseDir() throws Exception { User user = getUser(); + String fileName = "ResourcesServiceTest.jar"; when(userMapper.selectById(user.getId())).thenReturn(getUser()); when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn("/dolphinscheduler/123/resources/"); - try { - when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any())).thenReturn(getStorageEntityResource()); - } catch (Exception e) { - logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest", - e); - } + when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); + when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any())).thenReturn(getStorageEntityResource(fileName)); Result result = resourcesService.queryResourceBaseDir(user, ResourceType.FILE); assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @@ -648,25 +607,25 @@ public class ResourcesServiceTest { return user; } - private StorageEntity getStorageEntityResource() { + private StorageEntity getStorageEntityResource(String fileName) { StorageEntity entity = new StorageEntity(); - entity.setAlias("ResourcesServiceTest"); - entity.setFileName("ResourcesServiceTest"); + entity.setAlias(fileName); + entity.setFileName(fileName); entity.setDirectory(false); entity.setUserName(tenantCode); entity.setType(ResourceType.FILE); - entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest"); + entity.setFullName(tenantFileResourceDir + fileName); return entity; } - private StorageEntity getStorageEntityUdfResource() { + private StorageEntity getStorageEntityUdfResource(String fileName) { StorageEntity entity = new StorageEntity(); - entity.setAlias("ResourcesServiceTest1.jar"); - entity.setFileName("ResourcesServiceTest1.jar"); + entity.setAlias(fileName); + entity.setFileName(fileName); entity.setDirectory(false); entity.setUserName(tenantCode); entity.setType(ResourceType.UDF); - entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar"); + entity.setFullName(tenantUdfResourceDir + fileName); return entity; } From 920ac154cb9c0c8f33670af7c67387c0e30f6dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E8=80=90?= <46134044+sdhzwc@users.noreply.github.com> Date: Tue, 2 Apr 2024 09:42:44 +0800 Subject: [PATCH 29/88] [Improvement-15744][parameter] project parameter add update time and update user id (#15745) * project parameter add update time and update user id * project parameter add comment operator user id and UT * project parameter add ui --- .../service/impl/ProjectParameterServiceImpl.java | 2 ++ .../api/service/ProjectParameterServiceTest.java | 3 +++ .../dao/entity/ProjectParameter.java | 8 ++++++++ .../dao/mapper/ProjectParameterMapper.xml | 14 +++++++++----- .../src/main/resources/sql/dolphinscheduler_h2.sql | 1 + .../main/resources/sql/dolphinscheduler_mysql.sql | 1 + .../resources/sql/dolphinscheduler_postgresql.sql | 1 + .../3.2.2_schema/mysql/dolphinscheduler_ddl.sql | 1 + .../postgresql/dolphinscheduler_ddl.sql | 4 +++- dolphinscheduler-ui/src/locales/en_US/project.ts | 2 ++ dolphinscheduler-ui/src/locales/zh_CN/project.ts | 2 ++ .../src/views/projects/parameter/use-table.ts | 10 ++++++++++ 12 files changed, 43 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index 6e66f6286e..e30375e809 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -155,6 +155,8 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj projectParameter.setParamName(projectParameterName); projectParameter.setParamValue(projectParameterValue); + projectParameter.setUpdateTime(new Date()); + projectParameter.setOperator(loginUser.getId()); if (projectParameterMapper.updateById(projectParameter) > 0) { log.info("Project parameter is updated and id is :{}", projectParameter.getId()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index 4ab22bda21..7a3fb1b68d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -98,6 +98,9 @@ public class ProjectParameterServiceTest { Mockito.when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + ProjectParameter projectParameter = (ProjectParameter) result.getData(); + Assertions.assertNotNull(projectParameter.getOperator()); + Assertions.assertNotNull(projectParameter.getUpdateTime()); } @Test diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java index fbeeb387f1..03e9140145 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java @@ -42,6 +42,8 @@ public class ProjectParameter { @TableField("user_id") private Integer userId; + private Integer operator; + private long code; @TableField("project_code") @@ -56,4 +58,10 @@ public class ProjectParameter { private Date createTime; private Date updateTime; + + @TableField(exist = false) + private String createUser; + + @TableField(exist = false) + private String modifyUser; } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml index 159c263ea4..5b22d40a81 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml @@ -19,7 +19,7 @@ - id, param_name, param_value, code, project_code, user_id, create_time, update_time + id, param_name, param_value, code, project_code, user_id, operator, create_time, update_time select - - from t_ds_project_parameter + pp.id, param_name, param_value, code, project_code, user_id, operator, pp.create_time, pp.update_time, + u.user_name as create_user, + u2.user_name as modify_user + from t_ds_project_parameter pp + left join t_ds_user u on pp.user_id = u.id + left join t_ds_user u2 on pp.operator = u2.id where project_code = #{projectCode} - and id in + and pp.id in #{id} @@ -65,7 +69,7 @@ OR param_value LIKE concat('%', #{searchName}, '%') ) - order by update_time desc + order by pp.update_time desc - select + select t1.* from (select from t_ds_process_instance where process_definition_code=#{processDefinitionCode} and test_flag=#{testFlag} - + and schedule_time = ]]> #{startTime} and schedule_time #{endTime} + ) as t1 + + inner join + t_ds_task_instance as t2 + on t1.id = t2.process_instance_id and t2.task_code=#{taskDefinitionCode} order by end_time desc limit 1 diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java index 39b8d04e4d..0aae0dce2b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java @@ -263,7 +263,8 @@ public class ProcessInstanceMapperTest extends BaseDaoTest { processInstanceMapper.updateById(processInstance); ProcessInstance processInstance1 = - processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), null, null, + processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), 0L, null, + null, processInstance.getTestFlag()); Assertions.assertNotEquals(null, processInstance1); processInstanceMapper.deleteById(processInstance.getId()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java index 15ab34de34..28f9fd682b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java @@ -320,7 +320,7 @@ public class DependentExecute { int testFlag) { ProcessInstance lastSchedulerProcess = - processInstanceDao.queryLastSchedulerProcessInterval(definitionCode, dateInterval, testFlag); + processInstanceDao.queryLastSchedulerProcessInterval(definitionCode, taskCode, dateInterval, testFlag); ProcessInstance lastManualProcess = processInstanceDao.queryLastManualProcessInterval(definitionCode, taskCode, dateInterval, testFlag); From 5d8808dda40a5718df14a3ff703f07dfa253203c Mon Sep 17 00:00:00 2001 From: songwenyong <119404633+songwenyong@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:31:03 +0800 Subject: [PATCH 32/88] [Fix-15760][datasource-plugin] fix sql task split error (#15760) (#15794) * Fix the bug in SQL splitting by completing the task in two steps: 1. removeComment 2. split * Add a unit test for Hive SQL splitting. --- .../AbstractDataSourceProcessor.java | 3 ++- .../param/ClickHouseDataSourceProcessor.java | 3 ++- .../param/DamengDataSourceProcessor.java | 3 ++- .../db2/param/Db2DataSourceProcessor.java | 3 ++- .../hive/param/HiveDataSourceProcessor.java | 3 ++- .../param/HiveDataSourceProcessorTest.java | 20 +++++++++++++++++++ .../mysql/param/MySQLDataSourceProcessor.java | 3 ++- .../param/OceanBaseDataSourceProcessor.java | 3 ++- .../param/PostgreSQLDataSourceProcessor.java | 3 ++- .../param/SQLServerDataSourceProcessor.java | 3 ++- .../trino/param/TrinoDataSourceProcessor.java | 3 ++- 11 files changed, 40 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index db357e9d5c..98222a2c5a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -134,6 +134,7 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.other); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.other); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.other); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java index a613806460..81a4415f5d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java @@ -129,7 +129,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.clickhouse); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.clickhouse); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.clickhouse); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java index 1af61facd3..bc31bdcd49 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java @@ -135,7 +135,8 @@ public class DamengDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.dm); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.dm); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.dm); } private String transformOther(Map paramMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java index 2d67d91843..1d7c448355 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java @@ -129,7 +129,8 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.db2); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.db2); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.db2); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java index 36330c17c8..09d9f4b963 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java @@ -152,7 +152,8 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.hive); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.hive); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.hive); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java index 1da0cbadda..ef5e71729c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Assertions; @@ -94,4 +95,23 @@ public class HiveDataSourceProcessorTest { Assertions.assertEquals(DataSourceConstants.HIVE_VALIDATION_QUERY, hiveDatasourceProcessor.getValidationQuery()); } + + @Test + void splitAndRemoveComment() { + String sql = "create table if not exists test_ods.tb_test(\n" + + " `id` bigint COMMENT 'id', -- auto increment\n" + + " `user_name` string COMMENT 'username',\n" + + " `birthday` string COMMENT 'birthday',\n" + + " `gender` int COMMENT '1 male 2 female'\n" + + ") COMMENT 'user information table' PARTITIONED BY (`date_id` string);\n" + + "\n" + + "-- insert\n" + + "insert\n" + + " overwrite table test_ods.tb_test partition(date_id = '2024-03-28') -- partition\n" + + "values\n" + + " (1, 'Magic', '1990-10-01', '1');"; + List list = hiveDatasourceProcessor.splitAndRemoveComment(sql); + Assertions.assertEquals(list.size(), 2); + + } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java index b954defdd1..0c93b98212 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java @@ -177,7 +177,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.mysql); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.mysql); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.mysql); } private static boolean checkKeyIsLegitimate(String key) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java index b07b543c42..4d89f28eab 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java @@ -192,6 +192,7 @@ public class OceanBaseDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.oceanbase); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.oceanbase); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.oceanbase); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java index 2835d357ab..28a0aa2945 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java @@ -131,7 +131,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.postgresql); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.postgresql); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.postgresql); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java index 264f92c2b9..c3b73580dd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java @@ -128,7 +128,8 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.sqlserver); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.sqlserver); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.sqlserver); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java index 77b10b5162..bd53acaaf8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java @@ -131,7 +131,8 @@ public class TrinoDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.trino); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.trino); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.trino); } private String transformOther(Map otherMap) { From 98bc9ce4c9d4e161f48617ac0a1efeb10f9d6968 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Thu, 4 Apr 2024 20:29:18 +0800 Subject: [PATCH 33/88] feat: inc coverage of alert plugin instance svc (#15799) Co-authored-by: abzymeinsjtu --- .../ApiFuncIdentificationConstant.java | 3 +- .../impl/AlertPluginInstanceServiceImpl.java | 6 +- .../AlertPluginInstanceServiceTest.java | 107 +++++++++++++++++- 3 files changed, 105 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java index b93792307f..480272bdc6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java @@ -37,8 +37,7 @@ public class ApiFuncIdentificationConstant { public static final String TENANT_CREATE = "security:tenant:create"; public static final String TENANT_UPDATE = "security:tenant:update"; public static final String TENANT_DELETE = "security:tenant:delete"; - public static final String ALART_LIST = "monitor:alert:view"; - public static final String ALART_INSTANCE_CREATE = "security:alert-plugin:create"; + public static final String ALERT_INSTANCE_CREATE = "security:alert-plugin:create"; public static final String ALERT_PLUGIN_UPDATE = "security:alert-plugin:update"; public static final String ALERT_PLUGIN_DELETE = "security:alert-plugin:delete"; public static final String WORKER_GROUP_CREATE = "security:worker-group:create"; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java index 59eaca3c83..cf07c9fd73 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALART_INSTANCE_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_INSTANCE_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_UPDATE; @@ -108,7 +108,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A WarningType warningType, String pluginInstanceParams) { - if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALART_INSTANCE_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_INSTANCE_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -359,7 +359,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A throw new ServiceException(Status.ALERT_TEST_SENDING_FAILED, e.getMessage()); } - if (alertSendResponse.isSuccess()) { + if (!alertSendResponse.isSuccess()) { throw new ServiceException(Status.ALERT_TEST_SENDING_FAILED, alertSendResponse.getResResults().get(0).getMessage()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java index 52910858d5..bd8a039951 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.api.service; import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALART_INSTANCE_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALARM_INSTANCE_MANAGE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_INSTANCE_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_UPDATE; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.when; @@ -40,7 +42,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper; -import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -60,6 +61,9 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * alert plugin instance service test */ @@ -90,12 +94,16 @@ public class AlertPluginInstanceServiceTest { private User user; + private User noPermUser; + private final AlertPluginInstanceType normalInstanceType = AlertPluginInstanceType.NORMAL; private final AlertPluginInstanceType globalInstanceType = AlertPluginInstanceType.GLOBAL; private final WarningType warningType = WarningType.ALL; + private final Integer GLOBAL_ALERT_GROUP_ID = 2; + private String uiParams = "[\n" + " {\n" + " \"field\":\"userParams\",\n" @@ -172,21 +180,33 @@ public class AlertPluginInstanceServiceTest { private String paramsMap = "{\"path\":\"/kris/script/path\",\"userParams\":\"userParams\",\"type\":\"0\"}"; + private AlertPluginInstance alertPluginInstance; + @BeforeEach public void before() { user = new User(); user.setUserType(UserType.ADMIN_USER); user.setId(1); - AlertPluginInstance alertPluginInstance = getAlertPluginInstance(1, normalInstanceType, "test"); + + noPermUser = new User(); + noPermUser.setUserType(UserType.GENERAL_USER); + noPermUser.setId(2); + + alertPluginInstance = getAlertPluginInstance(1, normalInstanceType, "test"); alertPluginInstances = new ArrayList<>(); alertPluginInstances.add(alertPluginInstance); } @Test public void testCreate() { + when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, + noPermUser.getId(), ALERT_INSTANCE_CREATE, baseServiceLogger)).thenReturn(false); + assertThrowsServiceException(Status.USER_NO_OPERATION_PERM, () -> alertPluginInstanceService.create(noPermUser, + 1, "test", normalInstanceType, warningType, uiParams)); + when(alertPluginInstanceMapper.existInstanceName("test")).thenReturn(true); when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, - 1, ALART_INSTANCE_CREATE, baseServiceLogger)).thenReturn(true); + 1, ALERT_INSTANCE_CREATE, baseServiceLogger)).thenReturn(true); when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, null, 0, baseServiceLogger)).thenReturn(true); assertThrowsServiceException(Status.PLUGIN_INSTANCE_ALREADY_EXISTS, @@ -195,6 +215,19 @@ public class AlertPluginInstanceServiceTest { AlertPluginInstance alertPluginInstance = alertPluginInstanceService.create(user, 1, "test1", normalInstanceType, warningType, uiParams); assertNotNull(alertPluginInstance); + + when(alertGroupMapper.selectById(GLOBAL_ALERT_GROUP_ID)).thenReturn(getGlobalAlertGroup()); + assertDoesNotThrow(() -> alertPluginInstanceService.create(user, 1, "global_plugin_instance", + AlertPluginInstanceType.GLOBAL, warningType, uiParams)); + + when(alertGroupMapper.selectById(GLOBAL_ALERT_GROUP_ID)).thenReturn(getGlobalAlertGroup("1")); + assertDoesNotThrow(() -> alertPluginInstanceService.create(user, 1, "global_plugin_instance", + AlertPluginInstanceType.GLOBAL, warningType, uiParams)); + + when(alertPluginInstanceMapper.insert(Mockito.any())).thenReturn(-1); + assertThrowsServiceException(Status.SAVE_ERROR, + () -> alertPluginInstanceService.create(user, 1, "test_insert_error", normalInstanceType, warningType, + uiParams)); } @Test @@ -202,11 +235,10 @@ public class AlertPluginInstanceServiceTest { Mockito.when(registryClient.getServerList(RegistryNodeType.ALERT_SERVER)).thenReturn(new ArrayList<>()); assertThrowsServiceException(Status.ALERT_SERVER_NOT_EXIST, () -> alertPluginInstanceService.testSend(1, uiParams)); - AlertSendResponse.AlertSendResponseResult alertResult = new AlertSendResponse.AlertSendResponseResult(); - alertResult.setSuccess(true); Server server = new Server(); server.setPort(50052); server.setHost("127.0.0.1"); + Mockito.when(registryClient.getServerList(RegistryNodeType.ALERT_SERVER)) .thenReturn(Collections.singletonList(server)); assertThrowsServiceException(Status.ALERT_TEST_SENDING_FAILED, @@ -215,6 +247,11 @@ public class AlertPluginInstanceServiceTest { @Test public void testDelete() { + when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, + noPermUser.getId(), ALERT_PLUGIN_DELETE, baseServiceLogger)).thenReturn(false); + assertThrowsServiceException(Status.USER_NO_OPERATION_PERM, + () -> alertPluginInstanceService.deleteById(noPermUser, 1)); + List ids = Arrays.asList("11,2,3", "5,96", null, "98,1"); when(alertGroupMapper.queryInstanceIdsList()).thenReturn(ids); when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, @@ -241,10 +278,18 @@ public class AlertPluginInstanceServiceTest { when(alertPluginInstanceMapper.deleteById(5)).thenReturn(1); Assertions.assertDoesNotThrow(() -> alertPluginInstanceService.deleteById(user, 5)); + + when(alertGroupMapper.queryInstanceIdsList()).thenReturn(Collections.emptyList()); + Assertions.assertDoesNotThrow(() -> alertPluginInstanceService.deleteById(user, 9)); } @Test public void testUpdate() { + when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, + noPermUser.getId(), ALERT_PLUGIN_UPDATE, baseServiceLogger)).thenReturn(false); + assertThrowsServiceException(Status.USER_NO_OPERATION_PERM, + () -> alertPluginInstanceService.updateById(noPermUser, 1, "test", warningType, uiParams)); + when(alertPluginInstanceMapper.updateById(Mockito.any())).thenReturn(0); when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, 1, ALERT_PLUGIN_UPDATE, baseServiceLogger)).thenReturn(true); @@ -259,8 +304,51 @@ public class AlertPluginInstanceServiceTest { Assertions.assertNotNull(alertPluginInstance); } + @Test + public void testGetById() { + when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, + noPermUser.getId(), ALARM_INSTANCE_MANAGE, baseServiceLogger)).thenReturn(false); + assertThrowsServiceException(Status.USER_NO_OPERATION_PERM, + () -> alertPluginInstanceService.getById(noPermUser, 1)); + + when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, + user.getId(), ALARM_INSTANCE_MANAGE, baseServiceLogger)).thenReturn(true); + when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.ALERT_PLUGIN_INSTANCE, null, 0, + baseServiceLogger)).thenReturn(true); + when(alertPluginInstanceMapper.selectById(1)) + .thenReturn(getAlertPluginInstance(1, AlertPluginInstanceType.NORMAL, "test_get_instance")); + + Assertions.assertEquals(alertPluginInstanceService.getById(user, 1).getId(), 1); + } + + @Test + public void testCheckExistPluginInstanceName() { + when(alertPluginInstanceMapper.existInstanceName(Mockito.any(String.class))).thenReturn(false); + Assertions.assertEquals(false, alertPluginInstanceService.checkExistPluginInstanceName("test")); + } + + @Test + public void testListPaging() { + IPage page = new Page<>(); + page.setRecords(Collections.singletonList(alertPluginInstance)); + page.setTotal(1); + page.setPages(1); + + when(alertPluginInstanceMapper.queryByInstanceNamePage(Mockito.any(Page.class), Mockito.any(String.class))) + .thenReturn(page); + assertDoesNotThrow(() -> alertPluginInstanceService.listPaging(user, "test", 1, 1)); + } + @Test public void testQueryAll() { + when(alertPluginInstanceMapper.queryAllAlertPluginInstanceList()).thenReturn(Collections.emptyList()); + Assertions.assertEquals(0, alertPluginInstanceService.queryAll().size()); + + when(alertPluginInstanceMapper.queryAllAlertPluginInstanceList()) + .thenReturn(Collections.singletonList(alertPluginInstance)); + when(pluginDefineMapper.queryAllPluginDefineList()).thenReturn(Collections.emptyList()); + Assertions.assertEquals(0, alertPluginInstanceService.queryAll().size()); + AlertPluginInstance alertPluginInstance = getAlertPluginInstance(1, normalInstanceType, "test"); PluginDefine pluginDefine = new PluginDefine("script", "script", uiParams); pluginDefine.setId(1); @@ -283,4 +371,11 @@ public class AlertPluginInstanceServiceTest { return alertPluginInstance; } + private AlertGroup getGlobalAlertGroup(String... alertPluginInstanceIds) { + AlertGroup globalAlertGroup = new AlertGroup(); + globalAlertGroup.setId(2); + globalAlertGroup.setAlertInstanceIds(String.join(",", alertPluginInstanceIds)); + + return globalAlertGroup; + } } From 3b1de41acbee827b320370b14c6c73c7158f1aec Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 4 Apr 2024 23:44:12 +0800 Subject: [PATCH 34/88] Remove dolphinscheduler-data-quality from dolphinscheduler-task-dataquality (#15791) --- dolphinscheduler-api/pom.xml | 5 +++ .../quality/flow/batch/reader/JdbcReader.java | 8 +++- .../quality/flow/batch/writer/JdbcWriter.java | 8 +++- .../data/quality/utils/ParserUtilsTest.java | 39 ------------------- .../dolphinscheduler-task-dataquality/pom.xml | 5 --- .../plugin/task/dq/utils/RuleParserUtils.java | 23 +++++++---- 6 files changed, 32 insertions(+), 56 deletions(-) delete mode 100644 dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 681a85318a..40d556a17e 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -61,6 +61,11 @@ dolphinscheduler-meter + + org.apache.dolphinscheduler + dolphinscheduler-data-quality + + org.apache.dolphinscheduler dolphinscheduler-datasource-all diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java index 274d4f793a..97ae414051 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.data.quality.flow.batch.reader; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE; import static org.apache.dolphinscheduler.data.quality.Constants.DB_TABLE; import static org.apache.dolphinscheduler.data.quality.Constants.DOTS; @@ -32,17 +33,19 @@ import org.apache.dolphinscheduler.data.quality.config.ValidateResult; import org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment; import org.apache.dolphinscheduler.data.quality.flow.batch.BatchReader; import org.apache.dolphinscheduler.data.quality.utils.ConfigUtils; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.spark.sql.DataFrameReader; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import java.net.URLDecoder; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import lombok.SneakyThrows; + /** * AbstractJdbcSource */ @@ -74,6 +77,7 @@ public class JdbcReader implements BatchReader { return jdbcReader(env.sparkSession()).load(); } + @SneakyThrows private DataFrameReader jdbcReader(SparkSession sparkSession) { DataFrameReader reader = sparkSession.read() @@ -81,7 +85,7 @@ public class JdbcReader implements BatchReader { .option(URL, config.getString(URL)) .option(DB_TABLE, config.getString(DATABASE) + "." + config.getString(TABLE)) .option(USER, config.getString(USER)) - .option(PASSWORD, ParserUtils.decode(config.getString(PASSWORD))) + .option(PASSWORD, URLDecoder.decode(config.getString(PASSWORD), UTF_8.name())) .option(DRIVER, config.getString(DRIVER)); Config jdbcConfig = ConfigUtils.extractSubConfig(config, JDBC + DOTS, false); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java index 07b2bd60d5..b737567f21 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java @@ -33,13 +33,16 @@ import org.apache.dolphinscheduler.data.quality.config.Config; import org.apache.dolphinscheduler.data.quality.config.ValidateResult; import org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment; import org.apache.dolphinscheduler.data.quality.flow.batch.BatchWriter; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import lombok.SneakyThrows; + import com.google.common.base.Strings; /** @@ -70,6 +73,7 @@ public class JdbcWriter implements BatchWriter { } } + @SneakyThrows @Override public void write(Dataset data, SparkRuntimeEnvironment env) { if (!Strings.isNullOrEmpty(config.getString(SQL))) { @@ -82,7 +86,7 @@ public class JdbcWriter implements BatchWriter { .option(URL, config.getString(URL)) .option(DB_TABLE, config.getString(DATABASE) + "." + config.getString(TABLE)) .option(USER, config.getString(USER)) - .option(PASSWORD, ParserUtils.decode(config.getString(PASSWORD))) + .option(PASSWORD, URLDecoder.decode(config.getString(PASSWORD), StandardCharsets.UTF_8.name())) .mode(config.getString(SAVE_MODE)) .save(); } diff --git a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java b/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java deleted file mode 100644 index 328316cc39..0000000000 --- a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.data.quality.utils; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ParserUtilsTest { - - @Test - public void testParserUtils() { - String testStr = "aaa$bbb$ccc%ddd^eee#fff"; - String encode = ParserUtils.encode(testStr); - String decode = ParserUtils.decode(encode); - Assertions.assertEquals(testStr, decode); - - String blank = ""; - Assertions.assertEquals(ParserUtils.encode(blank), blank); - Assertions.assertEquals(ParserUtils.decode(blank), blank); - - Assertions.assertNull(ParserUtils.encode(null)); - Assertions.assertNull(ParserUtils.decode(null)); - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml index cc64b6cdcb..6626234a19 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml @@ -43,11 +43,6 @@ dolphinscheduler-datasource-all ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-data-quality - ${project.version} - org.apache.commons commons-collections4 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java index e99bb3dfaf..185573f66e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.dq.utils; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_BUSINESS_DATE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_CURRENT_DATE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_DATETIME; @@ -62,7 +63,6 @@ import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.USER; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; @@ -80,12 +80,15 @@ import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.SneakyThrows; + import com.fasterxml.jackson.databind.node.ArrayNode; /** @@ -102,9 +105,10 @@ public class RuleParserUtils { private static final String AND_TARGET_FILTER = "AND (${target_filter})"; private static final String WHERE_TARGET_FILTER = "WHERE (${target_filter})"; + @SneakyThrows public static List getReaderConfigList( Map inputParameterValue, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { List readerConfigList = new ArrayList<>(); @@ -123,7 +127,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getSourceType()), sourceDataSource)); config.put(USER, sourceDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(sourceDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(sourceDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getSourceType()))); String outputTable = inputParameterValue.get(SRC_DATABASE) + "_" + inputParameterValue.get(SRC_TABLE); @@ -150,7 +154,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()), targetDataSource)); config.put(USER, targetDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(targetDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(targetDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); String outputTable = @@ -264,9 +268,10 @@ public class RuleParserUtils { return defaultInputParameterValue; } + @SneakyThrows public static List getWriterConfigList( String sql, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getWriterConnectorType())) { @@ -284,7 +289,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()), writerDataSource)); config.put(USER, writerDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(writerDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); config.put(SQL, sql); @@ -336,8 +341,9 @@ public class RuleParserUtils { return readerConfigList; } + @SneakyThrows public static BaseConfig getStatisticsValueConfig( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { BaseConfig baseConfig = null; if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConnectionParam writerDataSource = @@ -354,7 +360,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl( DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()), writerDataSource)); config.put(USER, writerDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(writerDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); } @@ -544,6 +550,7 @@ public class RuleParserUtils { /** * the unique code use to get the same type and condition task statistics value + * * @param inputParameterValue * @return */ From 200d23fc3ebc67058fb49167b41b95c1eb62be8d Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Sun, 7 Apr 2024 09:58:28 +0800 Subject: [PATCH 35/88] [TEST] increase coverage of datasource service (#15801) Co-authored-by: abzymeinsjtu --- .../service/impl/DataSourceServiceImpl.java | 6 +- .../api/service/DataSourceServiceTest.java | 144 ++++++++++++++---- .../dao/mapper/DataSourceMapper.java | 3 +- 3 files changed, 121 insertions(+), 32 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index a030762457..210900e54c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -230,7 +230,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Override public PageInfo queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - IPage dataSourceList = null; + IPage dataSourceList; Page dataSourcePage = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { @@ -282,7 +282,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Override public List queryDataSourceList(User loginUser, Integer type) { - List datasourceList = null; + List datasourceList; if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { datasourceList = dataSourceMapper.queryDataSourceByType(0, type); } else { @@ -420,7 +420,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource public List getTables(Integer datasourceId, String database) { DataSource dataSource = dataSourceMapper.selectById(datasourceId); - List tableList = null; + List tableList; BaseConnectionParam connectionParam = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( dataSource.getType(), diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 68d4db0ff2..b4e7aae4b8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -52,15 +52,15 @@ import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.CollectionUtils; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Random; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; @@ -73,6 +73,9 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.dao.DuplicateKeyException; + +import com.baomidou.mybatisplus.core.metadata.IPage; /** * data source service test @@ -96,6 +99,15 @@ public class DataSourceServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; + @Mock + private IPage dataSourceList; + + private String randomStringWithLengthN(int n) { + byte[] bitArray = new byte[n]; + new Random().nextBytes(bitArray); + return new String(bitArray, StandardCharsets.UTF_8); + } + private void passResourcePermissionCheckService() { when(resourcePermissionCheckService.operationPermissionCheck(Mockito.any(), Mockito.anyInt(), Mockito.anyString(), Mockito.any())).thenReturn(true); @@ -131,6 +143,7 @@ public class DataSourceServiceTest { when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList); passResourcePermissionCheckService(); + // DATASOURCE_EXIST assertThrowsServiceException(Status.DATASOURCE_EXIST, () -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam)); @@ -140,13 +153,24 @@ public class DataSourceServiceTest { when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); + // DESCRIPTION TOO LONG + postgreSqlDatasourceParam.setNote(randomStringWithLengthN(512)); + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam)); + postgreSqlDatasourceParam.setNote(dataSourceDesc); + // SUCCESS assertDoesNotThrow(() -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam)); + + // Duplicated Key Exception + when(dataSourceMapper.insert(Mockito.any(DataSource.class))).thenThrow(DuplicateKeyException.class); + assertThrowsServiceException(Status.DATASOURCE_EXIST, + () -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam)); } } @Test - public void updateDataSourceTest() throws ExecutionException { + public void updateDataSourceTest() { User loginUser = getAdminUser(); int dataSourceId = 12; @@ -200,32 +224,74 @@ public class DataSourceServiceTest { // DATASOURCE_CONNECT_FAILED when(dataSourceMapper.queryDataSourceByName(postgreSqlDatasourceParam.getName())).thenReturn(null); + // DESCRIPTION TOO LONG + postgreSqlDatasourceParam.setNote(randomStringWithLengthN(512)); + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam)); + postgreSqlDatasourceParam.setNote(dataSourceDesc); + // SUCCESS assertDoesNotThrow(() -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam)); + + // Duplicated Key Exception + when(dataSourceMapper.updateById(Mockito.any(DataSource.class))).thenThrow(DuplicateKeyException.class); + assertThrowsServiceException(Status.DATASOURCE_EXIST, + () -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam)); } } @Test - public void queryDataSourceListPagingTest() { - User loginUser = getAdminUser(); + public void testQueryDataSourceListPaging() { + + User adminUser = getAdminUser(); + User generalUser = getGeneralUser(); String searchVal = ""; int pageNo = 1; int pageSize = 10; PageInfo pageInfo = - dataSourceService.queryDataSourceListPaging(loginUser, searchVal, pageNo, pageSize); + dataSourceService.queryDataSourceListPaging(adminUser, searchVal, pageNo, pageSize); Assertions.assertNotNull(pageInfo); + + // test query datasource as general user with no datasource authed + when(dataSourceList.getRecords()).thenReturn(getSingleDataSourceList()); + when(dataSourceMapper.selectPagingByIds(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(dataSourceList); + assertDoesNotThrow(() -> dataSourceService.queryDataSourceListPaging(generalUser, searchVal, pageNo, pageSize)); + + // test query datasource as general user with datasource authed + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, + generalUser.getId(), dataSourceServiceLogger)).thenReturn(Collections.singleton(1)); + + assertDoesNotThrow(() -> dataSourceService.queryDataSourceListPaging(generalUser, searchVal, pageNo, pageSize)); } @Test - public void connectionTest() { + public void testConnectionTest() { int dataSourceId = -1; when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); assertThrowsServiceException(Status.RESOURCE_NOT_EXIST, () -> dataSourceService.connectionTest(dataSourceId)); + + try ( + MockedStatic ignored = + Mockito.mockStatic(DataSourceUtils.class)) { + DataSource dataSource = getOracleDataSource(999); + when(dataSourceMapper.selectById(dataSource.getId())).thenReturn(dataSource); + DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class); + + when(DataSourceUtils.getDatasourceProcessor(Mockito.any())).thenReturn(dataSourceProcessor); + when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(true); + assertDoesNotThrow(() -> dataSourceService.connectionTest(dataSource.getId())); + + when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false); + assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE, + () -> dataSourceService.connectionTest(dataSource.getId())); + } + } @Test - public void deleteTest() { + public void testDelete() { User loginUser = getAdminUser(); int dataSourceId = 1; // resource not exist @@ -252,7 +318,7 @@ public class DataSourceServiceTest { } @Test - public void unauthDatasourceTest() { + public void testUnAuthDatasource() { User loginUser = getAdminUser(); loginUser.setId(1); loginUser.setUserType(UserType.ADMIN_USER); @@ -279,7 +345,7 @@ public class DataSourceServiceTest { } @Test - public void authedDatasourceTest() { + public void testAuthedDatasource() { User loginUser = getAdminUser(); loginUser.setId(1); loginUser.setUserType(UserType.ADMIN_USER); @@ -300,19 +366,28 @@ public class DataSourceServiceTest { } @Test - public void queryDataSourceListTest() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - Set dataSourceIds = new HashSet<>(); - dataSourceIds.add(1); + public void testQueryDataSourceList() { + User adminUser = getAdminUser(); + assertDoesNotThrow(() -> dataSourceService.queryDataSourceList(adminUser, DbType.MYSQL.ordinal())); + + User generalUser = getGeneralUser(); + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, - loginUser.getId(), dataSourceServiceLogger)).thenReturn(dataSourceIds); + generalUser.getId(), dataSourceServiceLogger)).thenReturn(Collections.emptySet()); + List emptyList = dataSourceService.queryDataSourceList(generalUser, DbType.MYSQL.ordinal()); + Assertions.assertEquals(emptyList.size(), 0); + + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, + generalUser.getId(), dataSourceServiceLogger)).thenReturn(Collections.singleton(1)); DataSource dataSource = new DataSource(); + dataSource.setId(1); dataSource.setType(DbType.MYSQL); - when(dataSourceMapper.selectBatchIds(dataSourceIds)).thenReturn(Collections.singletonList(dataSource)); + when(dataSourceMapper.selectBatchIds(Collections.singleton(1))) + .thenReturn(Collections.singletonList(dataSource)); + List list = - dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal()); + dataSourceService.queryDataSourceList(generalUser, DbType.MYSQL.ordinal()); Assertions.assertNotNull(list); } @@ -327,21 +402,28 @@ public class DataSourceServiceTest { } @Test - public void queryDataSourceTest() { - when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(null); + public void testQueryDataSource() { + // datasource not exists + when(dataSourceMapper.selectById(999)).thenReturn(null); User loginUser = new User(); loginUser.setUserType(UserType.GENERAL_USER); loginUser.setId(2); - try { - dataSourceService.queryDataSource(Mockito.anyInt(), loginUser); - } catch (Exception e) { - Assertions.assertTrue(e.getMessage().contains(Status.RESOURCE_NOT_EXIST.getMsg())); - } + + assertThrowsServiceException(Status.RESOURCE_NOT_EXIST, + () -> dataSourceService.queryDataSource(999, loginUser)); DataSource dataSource = getOracleDataSource(1); - when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(dataSource); + when(dataSourceMapper.selectById(dataSource.getId())).thenReturn(dataSource); when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.DATASOURCE, loginUser.getId(), DATASOURCE, baseServiceLogger)).thenReturn(true); + + // no perm + when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.DATASOURCE, + new Object[]{dataSource.getId()}, loginUser.getId(), baseServiceLogger)).thenReturn(false); + assertThrowsServiceException(Status.USER_NO_OPERATION_PERM, + () -> dataSourceService.queryDataSource(dataSource.getId(), loginUser)); + + // success when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.DATASOURCE, new Object[]{dataSource.getId()}, loginUser.getId(), baseServiceLogger)).thenReturn(true); BaseDataSourceParamDTO paramDTO = dataSourceService.queryDataSource(dataSource.getId(), loginUser); @@ -472,8 +554,16 @@ public class DataSourceServiceTest { */ private User getAdminUser() { User loginUser = new User(); - loginUser.setId(-1); + loginUser.setId(1); loginUser.setUserName("admin"); + loginUser.setUserType(UserType.ADMIN_USER); + return loginUser; + } + + private User getGeneralUser() { + User loginUser = new User(); + loginUser.setId(2); + loginUser.setUserName("user"); loginUser.setUserType(UserType.GENERAL_USER); return loginUser; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java index b5dcc31627..e26fe77ae4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java @@ -102,8 +102,7 @@ public interface DataSourceMapper extends BaseMapper { /** * selectPagingByIds * @param dataSourcePage - * @param ids - * @param searchVal + * @param dataSourceIds * @return */ IPage selectPagingByIds(Page dataSourcePage, From 39274094f8889ea0c21bc5b273052da8cd58a616 Mon Sep 17 00:00:00 2001 From: John Huang Date: Sun, 7 Apr 2024 15:28:21 +0800 Subject: [PATCH 36/88] Fix typo on common.properties (#15806) --- deploy/kubernetes/dolphinscheduler/templates/configmap.yaml | 2 +- .../templates/deployment-dolphinscheduler-alert.yaml | 2 +- .../templates/deployment-dolphinscheduler-api.yaml | 2 +- .../templates/statefulset-dolphinscheduler-master.yaml | 2 +- .../templates/statefulset-dolphinscheduler-worker.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml index 66ea53854b..c9af1c00a5 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml @@ -32,7 +32,7 @@ data: {{- end }} {{- end }} {{- end }} - common_properties: |- + common.properties: |- {{- if index .Values.conf "common" }} {{- range $key, $value := index .Values.conf "common" }} {{- if and $.Values.minio.enabled }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 41e01ef386..54316faeed 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -114,7 +114,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-alert - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties - subPath: common_properties + subPath: common.properties volumes: - name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.alert.persistentVolumeClaim.enabled }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index c2770cfd98..47bd4c3ca9 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -115,7 +115,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-api - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties - subPath: common_properties + subPath: common.properties {{- if .Values.api.taskTypeFilter.enabled }} - name: config-volume mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 888c35607e..66b4f8f8ee 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -112,7 +112,7 @@ spec: {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties - subPath: common_properties + subPath: common.properties {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-master diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 7a75849fae..231f943fa6 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -113,7 +113,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties - subPath: common_properties + subPath: common.properties {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }} From 9599b5aa815300a49d74a2513be3bfa11366742a Mon Sep 17 00:00:00 2001 From: songwenyong <119404633+songwenyong@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:57:52 +0800 Subject: [PATCH 37/88] =?UTF-8?q?fix=EF=BC=9ADataSource=20And=20UdfFunc=20?= =?UTF-8?q?list=20query=20use=20Enum=20code=20value=20not=20ordinal=20(#15?= =?UTF-8?q?714)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rick Cheng --- .../dolphinscheduler/api/controller/DataSourceController.java | 2 +- .../dolphinscheduler/api/controller/ResourcesController.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index 4d1f0f9f3d..d4fa83dd3a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -162,7 +162,7 @@ public class DataSourceController extends BaseController { @ApiException(QUERY_DATASOURCE_ERROR) public Result queryDataSourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") DbType type) { - List datasourceList = dataSourceService.queryDataSourceList(loginUser, type.ordinal()); + List datasourceList = dataSourceService.queryDataSourceList(loginUser, type.getCode()); return Result.success(datasourceList); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 773e734c22..e9c28a5f6e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -554,7 +554,7 @@ public class ResourcesController extends BaseController { @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR) public Result queryUdfFuncList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type) { - return udfFuncService.queryUdfFuncList(loginUser, type.ordinal()); + return udfFuncService.queryUdfFuncList(loginUser, type.getCode()); } /** From 66df5d4b907d74a5ba0d663f12222cb973f6ff5e Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 9 Apr 2024 14:04:24 +0800 Subject: [PATCH 38/88] Split cpuUsage to systemCpuUsage and jvmCpuUsage (#15803) --- deploy/kubernetes/dolphinscheduler/README.md | 8 +- .../kubernetes/dolphinscheduler/values.yaml | 16 +- docs/docs/en/architecture/configuration.md | 234 +++++++++--------- docs/docs/zh/architecture/configuration.md | 174 +++++++------ .../alert/registry/AlertHeartbeatTask.java | 3 +- .../common/model/AlertServerHeartBeat.java | 28 +-- .../common/model/BaseHeartBeat.java | 46 ++++ .../common/model/HeartBeat.java | 4 - .../common/model/MasterHeartBeat.java | 28 +-- .../common/model/WorkerHeartBeat.java | 29 +-- .../server/master/MasterServer.java | 2 +- .../config/MasterServerLoadProtection.java | 50 +--- .../master/metrics/MasterServerMetrics.java | 2 +- .../master/registry/MasterSlotManager.java | 3 +- .../master/registry/ServerNodeManager.java | 4 +- .../master/task/MasterHeartBeatTask.java | 3 +- .../src/main/resources/application.yaml | 8 +- .../master/config/MasterConfigTest.java | 31 ++- .../MasterServerLoadProtectionTest.java | 3 +- .../src/test/resources/application.yaml | 164 ++++++++++++ .../src/test/resources/logback.xml | 8 +- .../metrics/BaseServerLoadProtection.java | 67 +++++ .../meter/metrics/DefaultMetricsProvider.java | 11 +- .../meter/metrics/ServerLoadProtection.java | 24 ++ .../meter/metrics/SystemMetrics.java | 3 +- .../src/main/resources/application.yaml | 16 +- .../server/worker/WorkerServer.java | 2 +- .../config/WorkerServerLoadProtection.java | 50 +--- .../worker/task/WorkerHeartBeatTask.java | 3 +- .../src/main/resources/application.yaml | 8 +- .../WorkerServerLoadProtectionTest.java | 3 +- 31 files changed, 605 insertions(+), 430 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeat.java create mode 100644 dolphinscheduler-master/src/test/resources/application.yaml create mode 100644 dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/BaseServerLoadProtection.java create mode 100644 dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/ServerLoadProtection.java diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 5659605b95..33633f3b2e 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -200,9 +200,9 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | master.env.MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER | string | `"true"` | Master kill application when handle failover | | master.env.MASTER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Master max heartbeat interval | | master.env.MASTER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open master overload protection | -| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. | | master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. | -| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. | +| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. | +| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. | | master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. | | master.env.MASTER_STATE_WHEEL_INTERVAL | string | `"5s"` | master state wheel interval, the unit is second | | master.env.MASTER_TASK_COMMIT_INTERVAL | string | `"1s"` | master commit task interval, the unit is second | @@ -301,9 +301,9 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | worker.env.WORKER_HOST_WEIGHT | string | `"100"` | Worker host weight to dispatch tasks | | worker.env.WORKER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Worker heartbeat interval | | worker.env.WORKER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open worker overload protection | -| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. | | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. | -| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. | +| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. | +| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. | | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. | | worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. | | worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index a8d9a34875..98c2f70db0 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -508,10 +508,10 @@ master: MASTER_STATE_WHEEL_INTERVAL: "5s" # -- If set true, will open master overload protection MASTER_SERVER_LOAD_PROTECTION_ENABLED: false - # -- Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. - MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 - # -- Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. - MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7 + # -- Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 + # -- Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 # -- Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7 # -- Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. @@ -629,10 +629,10 @@ worker: env: # -- If set true, will open worker overload protection WORKER_SERVER_LOAD_PROTECTION_ENABLED: false - # -- Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. - WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 - # -- Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. - WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7 + # -- Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 + # -- Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7 # -- Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7 # -- Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index b9a26b865c..13d8932943 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -110,7 +110,8 @@ The directory structure of DolphinScheduler is as follows: dolphinscheduler-daemon.sh is responsible for DolphinScheduler startup and shutdown. Essentially, start-all.sh or stop-all.sh startup and shutdown the cluster via dolphinscheduler-daemon.sh. -Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical situation of resources. +Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical +situation of resources. Default simplified parameters are: @@ -128,44 +129,47 @@ export DOLPHINSCHEDULER_OPTS=" " ``` -> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to communicate). -> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6. +> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to +> communicate). +> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will +> use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6. ### Database connection related configuration DolphinScheduler uses Spring Hikari to manage database connections, configuration file location: -|Service| Configuration file | -|--|--| -|Master Server | `master-server/conf/application.yaml`| -|Api Server| `api-server/conf/application.yaml`| -|Worker Server| `worker-server/conf/application.yaml`| -|Alert Server| `alert-server/conf/application.yaml`| +| Service | Configuration file | +|---------------|---------------------------------------| +| Master Server | `master-server/conf/application.yaml` | +| Api Server | `api-server/conf/application.yaml` | +| Worker Server | `worker-server/conf/application.yaml` | +| Alert Server | `alert-server/conf/application.yaml` | The default configuration is as follows: -|Parameters | Default value| Description| -|--|--|--| -|spring.datasource.driver-class-name| org.postgresql.Driver |datasource driver| -|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |datasource connection url| -|spring.datasource.username|root|datasource username| -|spring.datasource.password|root|datasource password| -|spring.datasource.hikari.connection-test-query|select 1|validate connection by running the SQL| -|spring.datasource.hikari.minimum-idle| 5| minimum connection pool size number| -|spring.datasource.hikari.auto-commit|true|whether auto commit| -|spring.datasource.hikari.pool-name|DolphinScheduler|name of the connection pool| -|spring.datasource.hikari.maximum-pool-size|50| maximum connection pool size number| -|spring.datasource.hikari.connection-timeout|30000|connection timeout| -|spring.datasource.hikari.idle-timeout|600000|Maximum idle connection survival time| -|spring.datasource.hikari.leak-detection-threshold|0|Connection leak detection threshold| -|spring.datasource.hikari.initialization-fail-timeout|1|Connection pool initialization failed timeout| +| Parameters | Default value | Description | +|------------------------------------------------------|---------------------------------------------------|-----------------------------------------------| +| spring.datasource.driver-class-name | org.postgresql.Driver | datasource driver | +| spring.datasource.url | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | datasource connection url | +| spring.datasource.username | root | datasource username | +| spring.datasource.password | root | datasource password | +| spring.datasource.hikari.connection-test-query | select 1 | validate connection by running the SQL | +| spring.datasource.hikari.minimum-idle | 5 | minimum connection pool size number | +| spring.datasource.hikari.auto-commit | true | whether auto commit | +| spring.datasource.hikari.pool-name | DolphinScheduler | name of the connection pool | +| spring.datasource.hikari.maximum-pool-size | 50 | maximum connection pool size number | +| spring.datasource.hikari.connection-timeout | 30000 | connection timeout | +| spring.datasource.hikari.idle-timeout | 600000 | Maximum idle connection survival time | +| spring.datasource.hikari.leak-detection-threshold | 0 | Connection leak detection threshold | +| spring.datasource.hikari.initialization-fail-timeout | 1 | Connection pool initialization failed timeout | Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`. ### Zookeeper related configuration -DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions. Configuration file location: -|Service| Configuration file | +DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions. +Configuration file location: +|Service| Configuration file | |--|--| |Master Server | `master-server/conf/application.yaml`| |Api Server| `api-server/conf/application.yaml`| @@ -173,17 +177,17 @@ DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event m The default configuration is as follows: -|Parameters | Default value| Description| -|--|--|--| -|registry.zookeeper.namespace|dolphinscheduler|namespace of zookeeper| -|registry.zookeeper.connect-string|localhost:2181| the connection string of zookeeper| -|registry.zookeeper.retry-policy.base-sleep-time|60ms|time to wait between subsequent retries| -|registry.zookeeper.retry-policy.max-sleep|300ms|maximum time to wait between subsequent retries| -|registry.zookeeper.retry-policy.max-retries|5|maximum retry times| -|registry.zookeeper.session-timeout|30s|session timeout| -|registry.zookeeper.connection-timeout|30s|connection timeout| -|registry.zookeeper.block-until-connected|600ms|waiting time to block until the connection succeeds| -|registry.zookeeper.digest|{username}:{password}|digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) | +| Parameters | Default value | Description | +|-------------------------------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| registry.zookeeper.namespace | dolphinscheduler | namespace of zookeeper | +| registry.zookeeper.connect-string | localhost:2181 | the connection string of zookeeper | +| registry.zookeeper.retry-policy.base-sleep-time | 60ms | time to wait between subsequent retries | +| registry.zookeeper.retry-policy.max-sleep | 300ms | maximum time to wait between subsequent retries | +| registry.zookeeper.retry-policy.max-retries | 5 | maximum retry times | +| registry.zookeeper.session-timeout | 30s | session timeout | +| registry.zookeeper.connection-timeout | 30s | connection timeout | +| registry.zookeeper.block-until-connected | 600ms | waiting time to block until the connection succeeds | +| registry.zookeeper.digest | {username}:{password} | digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) | Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`. @@ -191,12 +195,12 @@ Note that DolphinScheduler also supports zookeeper related configuration through Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location: -|Service| Configuration file | -|--|--| -|Master Server | `master-server/conf/common.properties`| -|Api Server| `api-server/conf/common.properties`| -|Worker Server| `worker-server/conf/common.properties`| -|Alert Server| `alert-server/conf/common.properties`| +| Service | Configuration file | +|---------------|----------------------------------------| +| Master Server | `master-server/conf/common.properties` | +| Api Server | `api-server/conf/common.properties` | +| Worker Server | `worker-server/conf/common.properties` | +| Alert Server | `alert-server/conf/common.properties` | The default configuration is as follows: @@ -237,43 +241,43 @@ The default configuration is as follows: Location: `api-server/conf/application.yaml` -|Parameters | Default value| Description| -|--|--|--| -|server.port|12345|api service communication port| -|server.servlet.session.timeout|120m|session timeout| -|server.servlet.context-path|/dolphinscheduler/ |request path| -|spring.servlet.multipart.max-file-size|1024MB|maximum file size| -|spring.servlet.multipart.max-request-size|1024MB|maximum request size| -|server.jetty.max-http-post-size|5000000|jetty maximum post size| -|spring.banner.charset|UTF-8|message encoding| -|spring.jackson.time-zone|UTC|time zone| -|spring.jackson.date-format|"yyyy-MM-dd HH:mm:ss"|time format| -|spring.messages.basename|i18n/messages|i18n config| -|security.authentication.type|PASSWORD|authentication type| -|security.authentication.ldap.user.admin|read-only-admin|admin user account when you log-in with LDAP| -|security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls| -|security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn| -|security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username| -|security.authentication.ldap.password|password|LDAP password| -|security.authentication.ldap.user.identity-attribute|uid|LDAP user identity attribute| -|security.authentication.ldap.user.email-attribute|mail|LDAP user email attribute| -|security.authentication.ldap.user.not-exist-action|CREATE|action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY)| -|security.authentication.ldap.ssl.enable|false|LDAP ssl switch| -|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks file absolute path| -|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks password| -|security.authentication.casdoor.user.admin||admin user account when you log-in with Casdoor| -|casdoor.endpoint||Casdoor server url| -|casdoor.client-id||id in Casdoor| -|casdoor.client-secret||secret in Casdoor| -|casdoor.certificate||certificate in Casdoor| -|casdoor.organization-name||organization name in Casdoor| -|casdoor.application-name||application name in Casdoor| -|casdoor.redirect-url||doplhinscheduler login url| -|api.traffic.control.global.switch|false|traffic control global switch| -|api.traffic.control.max-global-qps-rate|300|global max request number per second| -|api.traffic.control.tenant-switch|false|traffic control tenant switch| -|api.traffic.control.default-tenant-qps-rate|10|default tenant max request number per second| -|api.traffic.control.customize-tenant-qps-rate||customize tenant max request number per second| +| Parameters | Default value | Description | +|-------------------------------------------------------|--------------------------------------|------------------------------------------------------------------------------------------------| +| server.port | 12345 | api service communication port | +| server.servlet.session.timeout | 120m | session timeout | +| server.servlet.context-path | /dolphinscheduler/ | request path | +| spring.servlet.multipart.max-file-size | 1024MB | maximum file size | +| spring.servlet.multipart.max-request-size | 1024MB | maximum request size | +| server.jetty.max-http-post-size | 5000000 | jetty maximum post size | +| spring.banner.charset | UTF-8 | message encoding | +| spring.jackson.time-zone | UTC | time zone | +| spring.jackson.date-format | "yyyy-MM-dd HH:mm:ss" | time format | +| spring.messages.basename | i18n/messages | i18n config | +| security.authentication.type | PASSWORD | authentication type | +| security.authentication.ldap.user.admin | read-only-admin | admin user account when you log-in with LDAP | +| security.authentication.ldap.urls | ldap://ldap.forumsys.com:389/ | LDAP urls | +| security.authentication.ldap.base.dn | dc=example,dc=com | LDAP base dn | +| security.authentication.ldap.username | cn=read-only-admin,dc=example,dc=com | LDAP username | +| security.authentication.ldap.password | password | LDAP password | +| security.authentication.ldap.user.identity-attribute | uid | LDAP user identity attribute | +| security.authentication.ldap.user.email-attribute | mail | LDAP user email attribute | +| security.authentication.ldap.user.not-exist-action | CREATE | action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY) | +| security.authentication.ldap.ssl.enable | false | LDAP ssl switch | +| security.authentication.ldap.ssl.trust-store | ldapkeystore.jks | LDAP jks file absolute path | +| security.authentication.ldap.ssl.trust-store-password | password | LDAP jks password | +| security.authentication.casdoor.user.admin | | admin user account when you log-in with Casdoor | +| casdoor.endpoint | | Casdoor server url | +| casdoor.client-id | | id in Casdoor | +| casdoor.client-secret | | secret in Casdoor | +| casdoor.certificate | | certificate in Casdoor | +| casdoor.organization-name | | organization name in Casdoor | +| casdoor.application-name | | application name in Casdoor | +| casdoor.redirect-url | | doplhinscheduler login url | +| api.traffic.control.global.switch | false | traffic control global switch | +| api.traffic.control.max-global-qps-rate | 300 | global max request number per second | +| api.traffic.control.tenant-switch | false | traffic control tenant switch | +| api.traffic.control.default-tenant-qps-rate | 10 | default tenant max request number per second | +| api.traffic.control.customize-tenant-qps-rate | | customize tenant max request number per second | ### Master Server related configuration @@ -292,9 +296,9 @@ Location: `master-server/conf/application.yaml` | master.task-commit-interval | 1000 | master commit task interval, the unit is millisecond | | master.state-wheel-interval | 5 | time to check status | | master.server-load-protection.enabled | true | If set true, will open master overload protection | -| master.server-load-protection.max-cpu-usage-percentage-thresholds | 0.7 | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. | -| master.server-load-protection.max-jvm-memory-usage-percentage-thresholds | 0.7 | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. | -| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. | +| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. | +| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Master max JVM cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. | +| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max system memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. | | master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. | | master.failover-interval | 10 | failover interval, the unit is minute | | master.kill-application-when-task-failover | true | whether to kill yarn/k8s application when failover taskInstance | @@ -306,23 +310,23 @@ Location: `master-server/conf/application.yaml` Location: `worker-server/conf/application.yaml` -| Parameters | Default value | Description | -|--------------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| worker.listen-port | 1234 | worker-service listen port | -| worker.exec-threads | 100 | worker-service execute thread number, used to limit the number of task instances in parallel | -| worker.max-heartbeat-interval | 10s | worker-service max heartbeat interval | -| worker.host-weight | 100 | worker host weight to dispatch tasks | -| worker.server-load-protection.enabled | true | If set true will open worker overload protection | -| worker.max-cpu-usage-percentage-thresholds.max-cpu-usage-percentage-thresholds | 0.7 | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. | -| worker.server-load-protection.max-jvm-memory-usage-percentage-thresholds | 0.7 | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. | -| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. | -| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. | -| worker.registry-disconnect-strategy.strategy | stop | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting | -| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely | -| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution | -| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. | -| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants | -| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. | +| Parameters | Default value | Description | +|-----------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| worker.listen-port | 1234 | worker-service listen port | +| worker.exec-threads | 100 | worker-service execute thread number, used to limit the number of task instances in parallel | +| worker.max-heartbeat-interval | 10s | worker-service max heartbeat interval | +| worker.host-weight | 100 | worker host weight to dispatch tasks | +| worker.server-load-protection.enabled | true | If set true will open worker overload protection | +| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, master server can execute workflow. | +| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Worker max JVM cpu usage, when the worker's jvm cpu usage is smaller then this value, master server can execute workflow. | +| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Worker max system memory usage , when the worker's system memory usage is smaller then this value, master server can execute workflow. | +| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Worker max disk usage , when the worker's disk usage is smaller then this value, master server can execute workflow. | +| worker.registry-disconnect-strategy.strategy | stop | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting | +| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely | +| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution | +| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. | +| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants | +| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. | ### Alert Server related configuration @@ -337,10 +341,10 @@ Location: `alert-server/conf/application.yaml` This part describes quartz configs and configure them based on your practical situation and resources. -|Service| Configuration file | -|--|--| -|Master Server | `master-server/conf/application.yaml`| -|Api Server| `api-server/conf/application.yaml`| +| Service | Configuration file | +|---------------|---------------------------------------| +| Master Server | `master-server/conf/application.yaml` | +| Api Server | `api-server/conf/application.yaml` | The default configuration is as follows: @@ -358,7 +362,8 @@ The default configuration is as follows: | spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate | | spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000 | -The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool configuration is different. +The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool +configuration is different. The default quartz threadpool configuration in *Master Server* is as follows: @@ -369,7 +374,8 @@ The default quartz threadpool configuration in *Master Server* is as follows: | spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 | | spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool | -Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured as `QuartzZeroSizeThreadPool` which has zero thread; +Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured +as `QuartzZeroSizeThreadPool` which has zero thread; The default configuration is as follows: | Parameters | Default value | @@ -378,7 +384,8 @@ The default configuration is as follows: ### dolphinscheduler_env.sh [load environment variables configs] -When using shell to commit tasks, DolphinScheduler will export environment variables from `bin/env/dolphinscheduler_env.sh`. The +When using shell to commit tasks, DolphinScheduler will export environment variables +from `bin/env/dolphinscheduler_env.sh`. The mainly configuration including `JAVA_HOME` and other environment paths. ```bash @@ -406,9 +413,10 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec ### Log related configuration -|Service| Configuration file | -|--|--| -|Master Server | `master-server/conf/logback-spring.xml`| -|Api Server| `api-server/conf/logback-spring.xml`| -|Worker Server| `worker-server/conf/logback-spring.xml`| -|Alert Server| `alert-server/conf/logback-spring.xml`| +| Service | Configuration file | +|---------------|-----------------------------------------| +| Master Server | `master-server/conf/logback-spring.xml` | +| Api Server | `api-server/conf/logback-spring.xml` | +| Worker Server | `worker-server/conf/logback-spring.xml` | +| Alert Server | `alert-server/conf/logback-spring.xml` | + diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 0b3ea9bc5b..08fded19e0 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -130,38 +130,40 @@ export DOLPHINSCHEDULER_OPTS=" > 不建议设置"-XX:DisableExplicitGC" , DolphinScheduler使用Netty进行通讯,设置该参数,可能会导致内存泄漏. > ->> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true"将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6. +>> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true" +>> 将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6. ## 数据库连接相关配置 在DolphinScheduler中使用Spring Hikari对数据库连接进行管理，配置文件位置： -|服务名称| 配置文件 | -|--|--| -|Master Server | `master-server/conf/application.yaml`| -|Api Server| `api-server/conf/application.yaml`| -|Worker Server| `worker-server/conf/application.yaml`| -|Alert Server| `alert-server/conf/application.yaml`| +| 服务名称 | 配置文件 | +|---------------|---------------------------------------| +| Master Server | `master-server/conf/application.yaml` | +| Api Server | `api-server/conf/application.yaml` | +| Worker Server | `worker-server/conf/application.yaml` | +| Alert Server | `alert-server/conf/application.yaml` | 默认配置如下： -|参数 | 默认值| 描述| -|--|--|--| -|spring.datasource.driver-class-name| org.postgresql.Driver |数据库驱动| -|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |数据库连接地址| -|spring.datasource.username|root|数据库用户名| -|spring.datasource.password|root|数据库密码| -|spring.datasource.hikari.connection-test-query|select 1|检测连接是否有效的sql| -|spring.datasource.hikari.minimum-idle| 5|最小空闲连接池数量| -|spring.datasource.hikari.auto-commit|true|是否自动提交| -|spring.datasource.hikari.pool-name|DolphinScheduler|连接池名称| -|spring.datasource.hikari.maximum-pool-size|50|连接池最大连接数| -|spring.datasource.hikari.connection-timeout|30000|连接超时时长| -|spring.datasource.hikari.idle-timeout|600000|空闲连接存活最大时间| -|spring.datasource.hikari.leak-detection-threshold|0|连接泄露检测阈值| -|spring.datasource.hikari.initialization-fail-timeout|1|连接池初始化失败timeout| +| 参数 | 默认值 | 描述 | +|------------------------------------------------------|---------------------------------------------------|-----------------| +| spring.datasource.driver-class-name | org.postgresql.Driver | 数据库驱动 | +| spring.datasource.url | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | 数据库连接地址 | +| spring.datasource.username | root | 数据库用户名 | +| spring.datasource.password | root | 数据库密码 | +| spring.datasource.hikari.connection-test-query | select 1 | 检测连接是否有效的sql | +| spring.datasource.hikari.minimum-idle | 5 | 最小空闲连接池数量 | +| spring.datasource.hikari.auto-commit | true | 是否自动提交 | +| spring.datasource.hikari.pool-name | DolphinScheduler | 连接池名称 | +| spring.datasource.hikari.maximum-pool-size | 50 | 连接池最大连接数 | +| spring.datasource.hikari.connection-timeout | 30000 | 连接超时时长 | +| spring.datasource.hikari.idle-timeout | 600000 | 空闲连接存活最大时间 | +| spring.datasource.hikari.leak-detection-threshold | 0 | 连接泄露检测阈值 | +| spring.datasource.hikari.initialization-fail-timeout | 1 | 连接池初始化失败timeout | -DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名, 设置值即可。 +DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名, +设置值即可。 ## Zookeeper相关配置 @@ -174,17 +176,17 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功 默认配置如下： -|参数 |默认值| 描述| -|--|--|--| -|registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace| -|registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息| -|registry.zookeeper.retry-policy.base-sleep-time|60ms|基本重试时间差| -|registry.zookeeper.retry-policy.max-sleep|300ms|最大重试时间| -|registry.zookeeper.retry-policy.max-retries|5|最大重试次数| -|registry.zookeeper.session-timeout|30s|session超时时间| -|registry.zookeeper.connection-timeout|30s|连接超时时间| -|registry.zookeeper.block-until-connected|600ms|阻塞直到连接成功的等待时间| -|registry.zookeeper.digest|{用户名:密码}|如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档)| +| 参数 | 默认值 | 描述 | +|-------------------------------------------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| registry.zookeeper.namespace | dolphinscheduler | Zookeeper集群使用的namespace | +| registry.zookeeper.connect-string | localhost:2181 | Zookeeper集群连接信息 | +| registry.zookeeper.retry-policy.base-sleep-time | 60ms | 基本重试时间差 | +| registry.zookeeper.retry-policy.max-sleep | 300ms | 最大重试时间 | +| registry.zookeeper.retry-policy.max-retries | 5 | 最大重试次数 | +| registry.zookeeper.session-timeout | 30s | session超时时间 | +| registry.zookeeper.connection-timeout | 30s | 连接超时时间 | +| registry.zookeeper.block-until-connected | 600ms | 阻塞直到连接成功的等待时间 | +| registry.zookeeper.digest | {用户名:密码} | 如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档) | DolphinScheduler同样可以通过`bin/env/dolphinscheduler_env.sh`进行Zookeeper相关的配置。 @@ -200,8 +202,8 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 默认配置如下： -| 参数 | 默认值 | 描述 | -|-----------------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 参数 | 默认值 | 描述 | +|-----------------------------------------------|--------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | data.basedir.path | /tmp/dolphinscheduler | 本地工作目录,用于存放临时文件 | | resource.storage.type | NONE | 资源文件存储类型: HDFS,S3,OSS,GCS,ABS,NONE | | resource.upload.path | /dolphinscheduler | 资源文件存储路径 | @@ -279,48 +281,54 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 位置：`master-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -|--------------------------------------------------------|--------------|-----------------------------------------------------------------------------------| -| master.listen-port | 5678 | master监听端口 | -| master.fetch-command-num | 10 | master拉取command数量 | -| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | -| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | -| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | -| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | -| master.max-heartbeat-interval | 10s | master最大心跳间隔 | -| master.task-commit-retry-times | 5 | 任务重试次数 | -| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | -| master.state-wheel-interval | 5 | 轮询检查状态时间 | -| master.max-cpu-load-avg | 1 | master最大cpuload均值,只有高于系统cpuload均值时,master服务才能调度任务. 默认值为1: 会使用100%的CPU | -| master.reserved-memory | 0.3 | master预留内存,只有低于系统可用内存时,master服务才能调度任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流 | -| master.failover-interval | 10 | failover间隔，单位为分钟 | -| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | -| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | -| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, | -| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | -| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| 参数 | 默认值 | 描述 | +|-----------------------------------------------------------------------------|--------------|-----------------------------------------------------------------------------------------| +| master.listen-port | 5678 | master监听端口 | +| master.fetch-command-num | 10 | master拉取command数量 | +| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | +| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | +| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | +| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | +| master.max-heartbeat-interval | 10s | master最大心跳间隔 | +| master.task-commit-retry-times | 5 | 任务重试次数 | +| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | +| master.state-wheel-interval | 5 | 轮询检查状态时间 | +| master.server-load-protection.enabled | true | 是否开启系统保护策略 | +| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | +| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | +| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | +| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | +| master.failover-interval | 10 | failover间隔，单位为分钟 | +| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | +| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | +| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, | +| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | +| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | ## Worker Server相关配置 位置：`worker-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -|------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------| -| worker.listen-port | 1234 | worker监听端口 | -| worker.exec-threads | 100 | worker工作线程数量,用于限制并行的任务实例数量 | -| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 | -| worker.host-weight | 100 | 派发任务时，worker主机的权重 | -| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 | -| worker.max-cpu-load-avg | 1 | worker最大cpuload均值,只有高于系统cpuload均值时,worker服务才能被派发任务. 默认值为1: 会使用100%的CPU | -| worker.reserved-memory | 0.3 | worker预留内存,只有低于系统可用内存时,worker服务才能被派发任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流 | -| worker.alert-listen-host | localhost | alert监听host | -| worker.alert-listen-port | 50052 | alert监听端口 | -| worker.registry-disconnect-strategy.strategy | stop | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | -| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 | -| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 | -| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 | -| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 | -| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 | +| 参数 | 默认值 | 描述 | +|-----------------------------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------| +| worker.listen-port | 1234 | worker监听端口 | +| worker.exec-threads | 100 | worker工作线程数量,用于限制并行的任务实例数量 | +| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 | +| worker.host-weight | 100 | 派发任务时，worker主机的权重 | +| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 | +| worker.server-load-protection.enabled | true | 是否开启系统保护策略 | +| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | worker最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统CPU | +| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | worker最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的JVM CPU | +| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | worker最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统内存 | +| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | worker最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | +| worker.alert-listen-host | localhost | alert监听host | +| worker.alert-listen-port | 50052 | alert监听端口 | +| worker.registry-disconnect-strategy.strategy | stop | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | +| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 | +| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 | +| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 | +| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 | +| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 | ## Alert Server相关配置 @@ -366,7 +374,9 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 | | spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool | -因为*Api Server*不会启动*Quartz Scheduler*实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`不会启动任何线程。具体的默认配置如下： +因为*Api Server*不会启动*Quartz Scheduler* +实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool` +不会启动任何线程。具体的默认配置如下： | Parameters | Default value | |------------------------------------------------------|-----------------------------------------------------------------------| @@ -374,7 +384,8 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId ## dolphinscheduler_env.sh [环境变量配置] -通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME` 任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。 +通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME` +任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。 ```bash # JAVA_HOME, will use it to start DolphinScheduler server @@ -401,9 +412,10 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec ## 日志相关配置 -|服务名称| 配置文件 | -|--|--| -|Master Server | `master-server/conf/logback-spring.xml`| -|Api Server| `api-server/conf/logback-spring.xml`| -|Worker Server| `worker-server/conf/logback-spring.xml`| -|Alert Server| `alert-server/conf/logback-spring.xml`| +| 服务名称 | 配置文件 | +|---------------|-----------------------------------------| +| Master Server | `master-server/conf/logback-spring.xml` | +| Api Server | `api-server/conf/logback-spring.xml` | +| Worker Server | `worker-server/conf/logback-spring.xml` | +| Alert Server | `alert-server/conf/logback-spring.xml` | + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java index 3b2d588f85..0bfefed223 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java @@ -65,7 +65,8 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask .processId(processId) .startupTime(startupTime) .reportTime(System.currentTimeMillis()) - .cpuUsage(systemMetrics.getTotalCpuUsedPercentage()) + .jvmCpuUsage(systemMetrics.getJvmCpuUsagePercentage()) + .cpuUsage(systemMetrics.getSystemCpuUsagePercentage()) .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) .serverStatus(ServerStatus.NORMAL) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java index 7cbd83b8ce..9faaef82be 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java @@ -17,33 +17,11 @@ package org.apache.dolphinscheduler.common.model; -import org.apache.dolphinscheduler.common.enums.ServerStatus; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; -@Data -@Builder +@SuperBuilder @NoArgsConstructor -@AllArgsConstructor -public class AlertServerHeartBeat implements HeartBeat { +public class AlertServerHeartBeat extends BaseHeartBeat implements HeartBeat { - private int processId; - private long startupTime; - private long reportTime; - private double cpuUsage; - private double memoryUsage; - private double jvmMemoryUsage; - - private ServerStatus serverStatus; - - private String host; - private int port; - - @Override - public ServerStatus getServerStatus() { - return serverStatus; - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeat.java new file mode 100644 index 0000000000..2837e5482b --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeat.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.model; + +import org.apache.dolphinscheduler.common.enums.ServerStatus; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class BaseHeartBeat implements HeartBeat { + + protected int processId; + protected long startupTime; + protected long reportTime; + protected double jvmCpuUsage; + protected double cpuUsage; + protected double jvmMemoryUsage; + protected double memoryUsage; + protected double diskUsage; + protected ServerStatus serverStatus; + + protected String host; + protected int port; + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/HeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/HeartBeat.java index 3a105227aa..35971b398b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/HeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/HeartBeat.java @@ -21,10 +21,6 @@ import org.apache.dolphinscheduler.common.enums.ServerStatus; public interface HeartBeat { - String getHost(); - ServerStatus getServerStatus(); - int getPort(); - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java index ecc140bcfb..b8ae4512dd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java @@ -17,33 +17,11 @@ package org.apache.dolphinscheduler.common.model; -import org.apache.dolphinscheduler.common.enums.ServerStatus; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; -@Data -@Builder +@SuperBuilder @NoArgsConstructor -@AllArgsConstructor -public class MasterHeartBeat implements HeartBeat { +public class MasterHeartBeat extends BaseHeartBeat implements HeartBeat { - private long startupTime; - private long reportTime; - private double cpuUsage; - private double jvmMemoryUsage; - private double memoryUsage; - private double diskUsage; - private ServerStatus serverStatus; - private int processId; - - private String host; - private int port; - - @Override - public ServerStatus getServerStatus() { - return serverStatus; - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java index 056fc6a2c7..c027486198 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java @@ -17,37 +17,18 @@ package org.apache.dolphinscheduler.common.model; -import org.apache.dolphinscheduler.common.enums.ServerStatus; - -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; @Data -@Builder +@EqualsAndHashCode(callSuper = true) +@SuperBuilder @NoArgsConstructor -@AllArgsConstructor -public class WorkerHeartBeat implements HeartBeat { - - private long startupTime; - private long reportTime; - private double cpuUsage; - private double jvmMemoryUsage; - private double memoryUsage; - private double diskUsage; - private ServerStatus serverStatus; - private int processId; - - private String host; - private int port; +public class WorkerHeartBeat extends BaseHeartBeat implements HeartBeat { private int workerHostWeight; // worker host weight private int threadPoolUsage; // worker waiting task count - @Override - public ServerStatus getServerStatus() { - return serverStatus; - } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index d37ca9d016..752479e600 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -123,7 +123,7 @@ public class MasterServer implements IStoppable { MasterServerMetrics.registerMasterCpuUsageGauge(() -> { SystemMetrics systemMetrics = metricsProvider.getSystemMetrics(); - return systemMetrics.getTotalCpuUsedPercentage(); + return systemMetrics.getSystemCpuUsagePercentage(); }); MasterServerMetrics.registerMasterMemoryAvailableGauge(() -> { SystemMetrics systemMetrics = metricsProvider.getSystemMetrics(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtection.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtection.java index 03570d691d..6b259738fe 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtection.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtection.java @@ -17,57 +17,11 @@ package org.apache.dolphinscheduler.server.master.config; -import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.meter.metrics.BaseServerLoadProtection; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@Data -@NoArgsConstructor -@AllArgsConstructor -public class MasterServerLoadProtection { - - private boolean enabled = true; - - private double maxCpuUsagePercentageThresholds = 0.7; - - private double maxJVMMemoryUsagePercentageThresholds = 0.7; - - private double maxSystemMemoryUsagePercentageThresholds = 0.7; - - private double maxDiskUsagePercentageThresholds = 0.7; - - public boolean isOverload(SystemMetrics systemMetrics) { - if (!enabled) { - return false; - } - if (systemMetrics.getTotalCpuUsedPercentage() > maxCpuUsagePercentageThresholds) { - log.info( - "Master OverLoad: the TotalCpuUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", - systemMetrics.getTotalCpuUsedPercentage(), maxCpuUsagePercentageThresholds); - return true; - } - if (systemMetrics.getJvmMemoryUsedPercentage() > maxJVMMemoryUsagePercentageThresholds) { - log.info( - "Master OverLoad: the JvmMemoryUsedPercentage: {} is over then the MaxJVMMemoryUsagePercentageThresholds {}", - systemMetrics.getJvmMemoryUsedPercentage(), maxCpuUsagePercentageThresholds); - return true; - } - if (systemMetrics.getDiskUsedPercentage() > maxDiskUsagePercentageThresholds) { - log.info("Master OverLoad: the DiskUsedPercentage: {} is over then the MaxDiskUsagePercentageThresholds {}", - systemMetrics.getDiskUsedPercentage(), maxCpuUsagePercentageThresholds); - return true; - } - if (systemMetrics.getSystemMemoryUsedPercentage() > maxSystemMemoryUsagePercentageThresholds) { - log.info( - "Master OverLoad: the SystemMemoryUsedPercentage: {} is over then the MaxSystemMemoryUsagePercentageThresholds {}", - systemMetrics.getSystemMemoryUsedPercentage(), maxSystemMemoryUsagePercentageThresholds); - return true; - } - return false; - } +public class MasterServerLoadProtection extends BaseServerLoadProtection { } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java index 09ba1cb4ba..1fc92200df 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java @@ -51,7 +51,7 @@ public class MasterServerMetrics { public void registerMasterCpuUsageGauge(Supplier supplier) { Gauge.builder("ds.master.cpu.usage", supplier) - .description("worker cpu usage") + .description("master cpu usage") .register(Metrics.globalRegistry); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterSlotManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterSlotManager.java index 834f56c2a4..155b973117 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterSlotManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterSlotManager.java @@ -70,7 +70,8 @@ public class MasterSlotManager { public void notify(Map masterNodeInfo) { List serverList = masterNodeInfo.values().stream() .filter(heartBeat -> !heartBeat.getServerStatus().equals(ServerStatus.BUSY)) - .map(this::convertHeartBeatToServer).collect(Collectors.toList()); + .map(this::convertHeartBeatToServer) + .collect(Collectors.toList()); syncMasterNodes(serverList); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java index 11a994aacd..258acd8f6e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java @@ -249,7 +249,9 @@ public class ServerNodeManager implements InitializingBean { try { Map workerNodeMaps = registryClient.getServerMaps(RegistryNodeType.WORKER); for (Map.Entry entry : workerNodeMaps.entrySet()) { - workerNodeInfo.put(entry.getKey(), JSONUtils.parseObject(entry.getValue(), WorkerHeartBeat.class)); + String nodeAddress = entry.getKey(); + WorkerHeartBeat workerHeartBeat = JSONUtils.parseObject(entry.getValue(), WorkerHeartBeat.class); + workerNodeInfo.put(nodeAddress, workerHeartBeat); } } finally { workerGroupWriteLock.unlock(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java index e9b0970ed3..b7b5e7a21e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java @@ -64,7 +64,8 @@ public class MasterHeartBeatTask extends BaseHeartBeatTask { return MasterHeartBeat.builder() .startupTime(ServerLifeCycleManager.getServerStartupTime()) .reportTime(System.currentTimeMillis()) - .cpuUsage(systemMetrics.getTotalCpuUsedPercentage()) + .jvmCpuUsage(systemMetrics.getJvmCpuUsagePercentage()) + .cpuUsage(systemMetrics.getSystemCpuUsagePercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .diskUsage(systemMetrics.getDiskUsedPercentage()) diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index ce7b1df7dd..a85eadb59f 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -122,10 +122,10 @@ master: server-load-protection: # If set true, will open master overload protection enabled: true - # Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. - max-cpu-usage-percentage-thresholds: 0.7 - # Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. - max-jvm-memory-usage-percentage-thresholds: 0.7 + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.7 # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. max-system-memory-usage-percentage-thresholds: 0.7 # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java index ed982933d2..faab44cf85 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.server.master.config; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit.jupiter.SpringExtension; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@ActiveProfiles("master") -@ExtendWith(SpringExtension.class) +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; + +@AutoConfigureMockMvc @SpringBootTest(classes = MasterConfig.class) public class MasterConfigTest { @@ -36,6 +35,18 @@ public class MasterConfigTest { @Test public void getMasterDispatchTaskNumber() { int masterDispatchTaskNumber = masterConfig.getDispatchTaskNumber(); - Assertions.assertEquals(3, masterDispatchTaskNumber); + assertEquals(30, masterDispatchTaskNumber); + } + + @Test + public void getServerLoadProtection() { + MasterServerLoadProtection serverLoadProtection = masterConfig.getServerLoadProtection(); + assertTrue(serverLoadProtection.isEnabled()); + assertEquals(0.77, serverLoadProtection.getMaxSystemCpuUsagePercentageThresholds()); + assertEquals(0.77, serverLoadProtection.getMaxJvmCpuUsagePercentageThresholds()); + assertEquals(0.77, serverLoadProtection.getMaxJvmCpuUsagePercentageThresholds()); + assertEquals(0.77, serverLoadProtection.getMaxSystemMemoryUsagePercentageThresholds()); + assertEquals(0.77, serverLoadProtection.getMaxDiskUsagePercentageThresholds()); + } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtectionTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtectionTest.java index 90627f99d3..ce12eb1bd9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtectionTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterServerLoadProtectionTest.java @@ -30,7 +30,8 @@ class MasterServerLoadProtectionTest { SystemMetrics systemMetrics = SystemMetrics.builder() .jvmMemoryUsedPercentage(0.71) .systemMemoryUsedPercentage(0.71) - .totalCpuUsedPercentage(0.71) + .systemCpuUsagePercentage(0.71) + .jvmCpuUsagePercentage(0.71) .diskUsedPercentage(0.71) .build(); masterServerLoadProtection.setEnabled(false); diff --git a/dolphinscheduler-master/src/test/resources/application.yaml b/dolphinscheduler-master/src/test/resources/application.yaml new file mode 100644 index 0000000000..0dbe490af3 --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/application.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + cache: + # default enable cache, you can disable by `type: none` + type: none + cache-names: + - tenant + - user + - processDefinition + - processTaskRelation + - taskDefinition + caffeine: + spec: maximumSize=100,expireAfterWrite=300s,recordStats + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + minimum-idle: 5 + auto-commit: true + validation-timeout: 3000 + pool-name: DolphinScheduler + maximum-pool-size: 50 + connection-timeout: 30000 + idle-timeout: 600000 + leak-detection-threshold: 0 + initialization-fail-timeout: 1 + quartz: + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.threadPool.threadPriority: 5 + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.threadPool.makeThreadsDaemons: true + org.quartz.threadPool.threadCount: 25 + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + +master: + listen-port: 5678 + # master fetch command num + fetch-command-num: 10 + # master prepare execute thread number to limit handle commands in parallel + pre-exec-threads: 10 + # master execute thread number to limit process instances in parallel + exec-threads: 100 + # master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s. + dispatch-task-number: 30 + # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight + host-selector: lower_weight + # master heartbeat interval + max-heartbeat-interval: 10s + # master commit task retry times + task-commit-retry-times: 5 + # master commit task interval + task-commit-interval: 1s + state-wheel-interval: 5s + server-load-protection: + # If set true, will open master overload protection + enabled: true + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.77 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.77 + # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.77 + # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. + max-disk-usage-percentage-thresholds: 0.77 + # failover interval, the unit is minute + failover-interval: 10m + # kill yarn / k8s application when failover taskInstance, default true + kill-application-when-task-failover: true + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: waiting + # The max waiting time to reconnect to registry if you set the strategy to waiting + max-waiting-time: 100s + worker-group-refresh-interval: 10s + +server: + port: 5679 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +metrics: + enabled: true diff --git a/dolphinscheduler-master/src/test/resources/logback.xml b/dolphinscheduler-master/src/test/resources/logback.xml index deb791fae2..4470a46391 100644 --- a/dolphinscheduler-master/src/test/resources/logback.xml +++ b/dolphinscheduler-master/src/test/resources/logback.xml @@ -66,12 +66,6 @@ - - - - - - - + diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/BaseServerLoadProtection.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/BaseServerLoadProtection.java new file mode 100644 index 0000000000..fd12d3bb66 --- /dev/null +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/BaseServerLoadProtection.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.meter.metrics; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Data +public class BaseServerLoadProtection implements ServerLoadProtection { + + protected boolean enabled = true; + + protected double maxSystemCpuUsagePercentageThresholds = 0.7; + + protected double maxJvmCpuUsagePercentageThresholds = 0.7; + + protected double maxSystemMemoryUsagePercentageThresholds = 0.7; + + protected double maxDiskUsagePercentageThresholds = 0.7; + + @Override + public boolean isOverload(SystemMetrics systemMetrics) { + if (!enabled) { + return false; + } + if (systemMetrics.getSystemCpuUsagePercentage() > maxSystemCpuUsagePercentageThresholds) { + log.info( + "OverLoad: the system cpu usage: {} is over then the maxSystemCpuUsagePercentageThresholds {}", + systemMetrics.getSystemCpuUsagePercentage(), maxSystemCpuUsagePercentageThresholds); + return true; + } + if (systemMetrics.getJvmCpuUsagePercentage() > maxJvmCpuUsagePercentageThresholds) { + log.info( + "OverLoad: the jvm cpu usage: {} is over then the maxJvmCpuUsagePercentageThresholds {}", + systemMetrics.getJvmCpuUsagePercentage(), maxJvmCpuUsagePercentageThresholds); + return true; + } + if (systemMetrics.getDiskUsedPercentage() > maxDiskUsagePercentageThresholds) { + log.info("OverLoad: the DiskUsedPercentage: {} is over then the maxDiskUsagePercentageThresholds {}", + systemMetrics.getDiskUsedPercentage(), maxDiskUsagePercentageThresholds); + return true; + } + if (systemMetrics.getSystemMemoryUsedPercentage() > maxSystemMemoryUsagePercentageThresholds) { + log.info( + "OverLoad: the SystemMemoryUsedPercentage: {} is over then the maxSystemMemoryUsagePercentageThresholds {}", + systemMetrics.getSystemMemoryUsedPercentage(), maxSystemMemoryUsagePercentageThresholds); + return true; + } + return false; + } +} diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java index 0ce6ceb4a4..f1240a0541 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.meter.metrics; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import io.micrometer.core.instrument.MeterRegistry; @@ -27,8 +26,11 @@ import io.micrometer.core.instrument.MeterRegistry; @Component public class DefaultMetricsProvider implements MetricsProvider { - @Autowired - private MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; + + public DefaultMetricsProvider(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } private SystemMetrics systemMetrics; @@ -53,8 +55,7 @@ public class DefaultMetricsProvider implements MetricsProvider { systemMetrics = SystemMetrics.builder() .systemCpuUsagePercentage(systemCpuUsage) - .processCpuUsagePercentage(processCpuUsage) - .totalCpuUsedPercentage(systemCpuUsage + processCpuUsage) + .jvmCpuUsagePercentage(processCpuUsage) .jvmMemoryUsed(jvmMemoryUsed) .jvmMemoryMax(jvmMemoryMax) .jvmMemoryUsedPercentage(jvmMemoryUsed / jvmMemoryMax) diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/ServerLoadProtection.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/ServerLoadProtection.java new file mode 100644 index 0000000000..3385de891f --- /dev/null +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/ServerLoadProtection.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.meter.metrics; + +public interface ServerLoadProtection { + + boolean isOverload(SystemMetrics systemMetrics); + +} diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/SystemMetrics.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/SystemMetrics.java index dcffafb83d..6da8f8ca4e 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/SystemMetrics.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/SystemMetrics.java @@ -30,8 +30,7 @@ public class SystemMetrics { // CPU private double systemCpuUsagePercentage; - private double processCpuUsagePercentage; - private double totalCpuUsedPercentage; + private double jvmCpuUsagePercentage; // JVM-Memory // todo: get pod memory usage diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 8e58b89568..d26ea5a4e0 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -190,10 +190,10 @@ master: state-wheel-interval: 5s server-load-protection: enabled: true - # Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. - max-cpu-usage-percentage-thresholds: 0.9 - # Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. - max-jvm-memory-usage-percentage-thresholds: 0.9 + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.9 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.9 # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. max-system-memory-usage-percentage-thresholds: 0.9 # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. @@ -215,10 +215,10 @@ worker: host-weight: 100 server-load-protection: enabled: true - # Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. - max-cpu-usage-percentage-thresholds: 0.9 - # Worker max JVM memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. - max-jvm-memory-usage-percentage-thresholds: 0.9 + # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + max-system-cpu-usage-percentage-thresholds: 0.9 + # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + max-jvm-cpu-usage-percentage-thresholds: 0.9 # Worker max System memory usage , when the worker's system memory usage is smaller then this value, worker server can be dispatched tasks. max-system-memory-usage-percentage-thresholds: 0.9 # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 2420ae5253..86e755fc8a 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -94,7 +94,7 @@ public class WorkerServer implements IStoppable { WorkerServerMetrics.registerWorkerCpuUsageGauge(() -> { SystemMetrics systemMetrics = metricsProvider.getSystemMetrics(); - return systemMetrics.getTotalCpuUsedPercentage(); + return systemMetrics.getSystemCpuUsagePercentage(); }); WorkerServerMetrics.registerWorkerMemoryAvailableGauge(() -> { SystemMetrics systemMetrics = metricsProvider.getSystemMetrics(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtection.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtection.java index 6e68a71bf5..1a52100eb2 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtection.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtection.java @@ -17,57 +17,11 @@ package org.apache.dolphinscheduler.server.worker.config; -import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.meter.metrics.BaseServerLoadProtection; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -@Data @Slf4j -@NoArgsConstructor -@AllArgsConstructor -public class WorkerServerLoadProtection { - - private boolean enabled = true; - - private double maxCpuUsagePercentageThresholds = 0.7; - - private double maxJVMMemoryUsagePercentageThresholds = 0.7; - - private double maxSystemMemoryUsagePercentageThresholds = 0.7; - - private double maxDiskUsagePercentageThresholds = 0.7; - - public boolean isOverload(SystemMetrics systemMetrics) { - if (!enabled) { - return false; - } - if (systemMetrics.getTotalCpuUsedPercentage() > maxCpuUsagePercentageThresholds) { - log.info( - "Worker OverLoad: the TotalCpuUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", - systemMetrics.getTotalCpuUsedPercentage(), maxCpuUsagePercentageThresholds); - return true; - } - if (systemMetrics.getJvmMemoryUsedPercentage() > maxJVMMemoryUsagePercentageThresholds) { - log.info( - "Worker OverLoad: the JvmMemoryUsedPercentage: {} is over then the maxCpuUsagePercentageThresholds {}", - systemMetrics.getJvmMemoryUsedPercentage(), maxJVMMemoryUsagePercentageThresholds); - return true; - } - if (systemMetrics.getDiskUsedPercentage() > maxDiskUsagePercentageThresholds) { - log.info("Worker OverLoad: the DiskUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", - systemMetrics.getDiskUsedPercentage(), maxDiskUsagePercentageThresholds); - return true; - } - if (systemMetrics.getSystemMemoryUsedPercentage() > maxSystemMemoryUsagePercentageThresholds) { - log.info( - "Worker OverLoad: the SystemMemoryUsedPercentage: {} is over then the MaxSystemMemoryUsagePercentageThresholds {}", - systemMetrics.getSystemMemoryUsedPercentage(), maxSystemMemoryUsagePercentageThresholds); - return true; - } - return false; - } +public class WorkerServerLoadProtection extends BaseServerLoadProtection { } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java index 57349e1448..4eefd9df10 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java @@ -65,7 +65,8 @@ public class WorkerHeartBeatTask extends BaseHeartBeatTask { return WorkerHeartBeat.builder() .startupTime(ServerLifeCycleManager.getServerStartupTime()) .reportTime(System.currentTimeMillis()) - .cpuUsage(systemMetrics.getTotalCpuUsedPercentage()) + .jvmCpuUsage(systemMetrics.getJvmCpuUsagePercentage()) + .cpuUsage(systemMetrics.getSystemCpuUsagePercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .diskUsage(systemMetrics.getDiskUsedPercentage()) diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index ad0535ac76..4361e8f014 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -50,10 +50,10 @@ worker: server-load-protection: # If set true, will open worker overload protection enabled: true - # Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. - max-cpu-usage-percentage-thresholds: 0.7 - # Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. - max-jvm-memory-usage-percentage-thresholds: 0.7 + # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + max-jvm-cpu-usage-percentage-thresholds: 0.7 # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. max-system-memory-usage-percentage-thresholds: 0.7 # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtectionTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtectionTest.java index 696e9c2478..204deb120e 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtectionTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/WorkerServerLoadProtectionTest.java @@ -30,7 +30,8 @@ class WorkerServerLoadProtectionTest { SystemMetrics systemMetrics = SystemMetrics.builder() .jvmMemoryUsedPercentage(0.71) .systemMemoryUsedPercentage(0.71) - .totalCpuUsedPercentage(0.71) + .systemCpuUsagePercentage(0.71) + .jvmCpuUsagePercentage(0.71) .diskUsedPercentage(0.71) .build(); workerServerLoadProtection.setEnabled(false); From 5050a8e1e680b4f0debf6ce59a22bd50afb7cb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Tue, 9 Apr 2024 14:38:47 +0800 Subject: [PATCH 39/88] [Improve] Fix typo on ProcessServiceImpl (#15817) --- .../dolphinscheduler/service/process/ProcessServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 261637529a..2b3dbbebbb 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -798,7 +798,7 @@ public class ProcessServiceImpl implements ProcessService { // recover tolerance fault process // If the workflow instance is in ready state, we will change to running, this can avoid the workflow // instance - // status is not correct with taskInsatnce status + // status is not correct with taskInstance status if (processInstance.getState() == WorkflowExecutionStatus.READY_PAUSE || processInstance.getState() == WorkflowExecutionStatus.READY_STOP) { // todo: If we handle the ready state in WorkflowExecuteRunnable then we can remove below code From f93b27e1217cab05e93953b71e5d2ea5e40fb8c8 Mon Sep 17 00:00:00 2001 From: sleo <97011595+alei1206@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:13:52 +0800 Subject: [PATCH 40/88] [Worker] Fix will not kill the subprocess in remote when stop a remote-shell task #15570 (#15629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix cannot kill the subprocess when stop a remote-shell task * move parse pid logic into ProcessUtils * extract common logic --------- Co-authored-by: 旺阳 Co-authored-by: Rick Cheng --- .../plugin/task/api/utils/ProcessUtils.java | 43 +++++++++++------- .../task/remoteshell/RemoteExecutor.java | 44 ++++++++++++++++++- .../task/remoteshell/RemoteExecutorTest.java | 19 ++++++++ 3 files changed, 89 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java index 7b61a1eaec..e8e31faa6d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java @@ -39,6 +39,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -117,33 +118,45 @@ public final class ProcessUtils { * @throws Exception exception */ public static String getPidsStr(int processId) throws Exception { - StringBuilder sb = new StringBuilder(); - Matcher mat = null; + + String rawPidStr; + // pstree pid get sub pids if (SystemUtils.IS_OS_MAC) { - String pids = OSUtils.exeCmd(String.format("%s -sp %d", TaskConstants.PSTREE, processId)); - if (StringUtils.isNotEmpty(pids)) { - mat = MACPATTERN.matcher(pids); + rawPidStr = OSUtils.exeCmd(String.format("%s -sp %d", TaskConstants.PSTREE, processId)); + } else if (SystemUtils.IS_OS_LINUX) { + rawPidStr = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); + } else { + rawPidStr = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); + } + + return parsePidStr(rawPidStr); + } + + public static String parsePidStr(String rawPidStr) { + + log.info("prepare to parse pid, raw pid string: {}", rawPidStr); + ArrayList allPidList = new ArrayList<>(); + Matcher mat = null; + if (SystemUtils.IS_OS_MAC) { + if (StringUtils.isNotEmpty(rawPidStr)) { + mat = MACPATTERN.matcher(rawPidStr); } } else if (SystemUtils.IS_OS_LINUX) { - String pids = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); - if (StringUtils.isNotEmpty(pids)) { - mat = LINUXPATTERN.matcher(pids); + if (StringUtils.isNotEmpty(rawPidStr)) { + mat = LINUXPATTERN.matcher(rawPidStr); } } else { - String pids = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); - if (StringUtils.isNotEmpty(pids)) { - mat = WINDOWSPATTERN.matcher(pids); + if (StringUtils.isNotEmpty(rawPidStr)) { + mat = WINDOWSPATTERN.matcher(rawPidStr); } } - if (null != mat) { while (mat.find()) { - sb.append(mat.group(1)).append(" "); + allPidList.add(mat.group(1)); } } - - return sb.toString().trim(); + return String.join(" ", allPidList).trim(); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java index c590fa9e44..334ebb6195 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java @@ -17,12 +17,16 @@ package org.apache.dolphinscheduler.plugin.task.remoteshell; +import static org.apache.dolphinscheduler.plugin.task.remoteshell.RemoteExecutor.COMMAND.PSTREE_COMMAND; + import org.apache.dolphinscheduler.plugin.datasource.ssh.SSHUtils; import org.apache.dolphinscheduler.plugin.datasource.ssh.param.SSHConnectionParam; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.parser.TaskOutputParameterParser; +import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.channel.ChannelExec; import org.apache.sshd.client.channel.ClientChannelEvent; @@ -50,7 +54,6 @@ public class RemoteExecutor implements AutoCloseable { static final int TRACK_INTERVAL = 5000; protected Map taskOutputParams = new HashMap<>(); - private SshClient sshClient; private ClientSession session; private SSHConnectionParam sshConnectionParam; @@ -154,11 +157,45 @@ public class RemoteExecutor implements AutoCloseable { public void kill(String taskId) throws IOException { String pid = getTaskPid(taskId); - String killCommand = String.format(COMMAND.KILL_COMMAND, pid); + + if (StringUtils.isEmpty(pid)) { + log.warn("query remote-shell task remote process id with empty"); + return; + } + if (!NumberUtils.isParsable(pid)) { + log.error("query remote-shell task remote process id error, pid {} can not parse to number", pid); + return; + } + + // query all pid + String remotePidStr = getAllRemotePidStr(pid); + String killCommand = String.format(COMMAND.KILL_COMMAND, remotePidStr); + log.info("prepare to execute kill command in host: {}, kill cmd: {}", sshConnectionParam.getHost(), + killCommand); runRemote(killCommand); cleanData(taskId); } + protected String getAllRemotePidStr(String pid) { + + String remoteProcessIdStr = ""; + String cmd = String.format(PSTREE_COMMAND, pid); + log.info("query all process id cmd: {}", cmd); + + try { + String rawPidStr = runRemote(cmd); + remoteProcessIdStr = ProcessUtils.parsePidStr(rawPidStr); + if (!remoteProcessIdStr.startsWith(pid)) { + log.error("query remote process id error, [{}] first pid not equal [{}]", remoteProcessIdStr, pid); + remoteProcessIdStr = pid; + } + } catch (Exception e) { + log.error("query remote all process id error", e); + remoteProcessIdStr = pid; + } + return remoteProcessIdStr; + } + public String getTaskPid(String taskId) throws IOException { String pidCommand = String.format(COMMAND.GET_PID_COMMAND, taskId); return runRemote(pidCommand).trim(); @@ -238,6 +275,9 @@ public class RemoteExecutor implements AutoCloseable { static final String ADD_STATUS_COMMAND = "\necho %s$?"; static final String CAT_FINAL_SCRIPT = "cat %s%s.sh"; + + static final String PSTREE_COMMAND = "pstree -p %s"; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/test/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutorTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/test/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutorTest.java index 975f059695..3cc9757ce1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/test/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutorTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/test/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutorTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.remoteshell; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -135,4 +136,22 @@ public class RemoteExecutorTest { doReturn("DOLPHINSCHEDULER-REMOTE-SHELL-TASK-STATUS-1").when(remoteExecutor).runRemote(trackCommand); Assertions.assertEquals(1, remoteExecutor.getTaskExitCode(taskId)); } + + @Test + void getAllRemotePidStr() throws IOException { + + RemoteExecutor remoteExecutor = spy(new RemoteExecutor(sshConnectionParam)); + doReturn("bash(9527)───sleep(9528)").when(remoteExecutor).runRemote(anyString()); + String allPidStr = remoteExecutor.getAllRemotePidStr("9527"); + Assertions.assertEquals("9527 9528", allPidStr); + + doReturn("systemd(1)───sleep(9528)").when(remoteExecutor).runRemote(anyString()); + allPidStr = remoteExecutor.getAllRemotePidStr("9527"); + Assertions.assertEquals("9527", allPidStr); + + doThrow(new TaskException()).when(remoteExecutor).runRemote(anyString()); + allPidStr = remoteExecutor.getAllRemotePidStr("9527"); + Assertions.assertEquals("9527", allPidStr); + + } } From 2b5f8433b732eee8c31a05f741636a007d5f2344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=A7=E5=9F=8E?= Date: Thu, 11 Apr 2024 14:13:53 +0800 Subject: [PATCH 41/88] Fix cannot construct instance of StreamingTaskTriggerResponse --- .../master/transportor/StreamingTaskTriggerResponse.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java index 0f9f265280..d25611aee0 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java @@ -19,9 +19,11 @@ package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class StreamingTaskTriggerResponse { private boolean success; From 883848f6ee00843d3c9e4668d607336d544dd110 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 11 Apr 2024 19:28:00 +0800 Subject: [PATCH 42/88] Remove unused caffeine cache (#15830) --- dolphinscheduler-master/pom.xml | 4 ---- .../src/main/resources/application.yaml | 11 ----------- .../src/test/resources/application.yaml | 11 ----------- .../src/main/resources/application.yaml | 11 ----------- 4 files changed, 37 deletions(-) diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 31627d4b7f..1b99dd97f3 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -102,10 +102,6 @@ org.codehaus.janino janino - - com.github.ben-manes.caffeine - caffeine - org.apache.hbase.thirdparty diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index a85eadb59f..c2d8f5e787 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -22,17 +22,6 @@ spring: jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats datasource: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler diff --git a/dolphinscheduler-master/src/test/resources/application.yaml b/dolphinscheduler-master/src/test/resources/application.yaml index 0dbe490af3..f4827d4b3c 100644 --- a/dolphinscheduler-master/src/test/resources/application.yaml +++ b/dolphinscheduler-master/src/test/resources/application.yaml @@ -20,17 +20,6 @@ spring: jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats datasource: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index d26ea5a4e0..654113471d 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -23,17 +23,6 @@ spring: date-format: "yyyy-MM-dd HH:mm:ss" banner: charset: UTF-8 - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats sql: init: schema-locations: classpath:sql/dolphinscheduler_h2.sql From e5e77492518a198877171801c1cf484b86f852e3 Mon Sep 17 00:00:00 2001 From: BaiJv Date: Fri, 12 Apr 2024 10:06:32 +0800 Subject: [PATCH 43/88] [Improvement] Abnormal characters check (#15824) * abnormal characters check * add test case * remove error log * fix code style * fix import --- .../service/impl/ResourcesServiceImpl.java | 5 +++++ .../api/utils/CheckUtils.java | 10 ++++++++++ .../api/utils/CheckUtilsTest.java | 20 +++++++++++++++++++ .../common/constants/Constants.java | 5 +++++ 4 files changed, 40 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 6a15da17a8..1c039cdfbd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkFilePath; import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS; import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; @@ -1290,6 +1291,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { return; } + // abnormal characters check + if (!checkFilePath(fullName)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH); + } // Avoid returning to the parent directory if (fullName.contains("../")) { throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 8b166a16dd..b394d4956c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -158,4 +158,14 @@ public class CheckUtils { return pattern.matcher(str).matches(); } + + /** + * regex FilePath check,only use a to z, A to Z, 0 to 9, and _./- + * + * @param str input string + * @return true if regex pattern is right, otherwise return false + */ + public static boolean checkFilePath(String str) { + return regexChecks(str, Constants.REGEX_FILE_PATH); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index bca8a69a16..da5ea88c83 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -92,4 +92,24 @@ public class CheckUtilsTest { Assertions.assertTrue(CheckUtils.checkPhone("17362537263")); } + /** + * check file path + */ + @Test + public void testCheckFilePath() { + // true + Assertions.assertTrue(CheckUtils.checkFilePath("/")); + Assertions.assertTrue(CheckUtils.checkFilePath("xx/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/xx")); + Assertions.assertTrue(CheckUtils.checkFilePath("14567134578654")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/1531531..13513/153135..")); + // false + Assertions.assertFalse(CheckUtils.checkFilePath(null)); + Assertions.assertFalse(CheckUtils.checkFilePath("file://xxx/ss")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss;/dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss && /dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss || /dasd/123")); + } + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 054a9410d5..19e1a1fabb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -252,6 +252,11 @@ public final class Constants { */ public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,39}$"); + /** + * file path regex + */ + public static final Pattern REGEX_FILE_PATH = Pattern.compile("^[a-zA-Z0-9_./-]+$"); + /** * read permission */ From 08ac1322864edf42903c7c03942fcad62c37da35 Mon Sep 17 00:00:00 2001 From: BaiJv Date: Fri, 12 Apr 2024 14:28:13 +0800 Subject: [PATCH 44/88] [Improvement] Modify python-gateway: enabled default to false. (#15825) * remove python-gateway:auth-token,modify python gateway: enabled default to false. * reset token --- dolphinscheduler-api/src/main/resources/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index c93fd99a59..61f9c8a592 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -149,8 +149,8 @@ api: #tenant1: 11 #tenant2: 20 python-gateway: - # Weather enable python gateway server or not. The default value is true. - enabled: true + # Weather enable python gateway server or not. The default value is false. + enabled: false # Authentication token for connection from python api to python gateway server. Should be changed the default value # when you deploy in public network. auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc From efbffacfbecd3632388e69ed400e3433ecc35f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Mon, 15 Apr 2024 11:29:11 +0800 Subject: [PATCH 45/88] delete useless code (#15844) --- .../dolphinscheduler/dao/mapper/UdfFuncMapper.java | 7 ------- .../dolphinscheduler/dao/mapper/UdfFuncMapper.xml | 12 ------------ .../dao/mapper/UdfFuncMapperTest.java | 13 ------------- 3 files changed, 32 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 6bc8049c7d..abc12f9aa1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -109,13 +109,6 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId, @Param("resourceIds") int[] resourceIds); - /** - * batch update udf func - * @param udfFuncList udf list - * @return update num - */ - int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); - /** * listAuthorizedUdfByUserId * @param userId diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index 24cb8de956..84f9fd9708 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -171,18 +171,6 @@ - - - update t_ds_udfs - - resource_name=#{udf.resourceName}, - update_time=#{udf.updateTime} - - - id=#{udf.id} - - - @@ -38,30 +38,25 @@ and log.time > #{startDate} and log.time #{endDate} - - and log.resource_type in - - #{i} + + and log.model_type in + + #{model_type} - - and log.operation in - - #{j} + + and log.operation_type in + + #{operation_type} - and u.user_name = #{userName} + and u.user_name like concat ('%', #{userName}, '%') + + + and log.model_name like concat ('%', #{modelName}, '%') order by log.time desc - diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 3c7b8805a0..4187b93a1c 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -2020,10 +2020,14 @@ CREATE TABLE t_ds_audit_log ( id int(11) NOT NULL AUTO_INCREMENT, user_id int(11) NOT NULL, - resource_type int(11) NOT NULL, - operation int(11) NOT NULL, + model_id bigint(20) NOT NULL, + model_name varchar(255) NOT NULL, + model_type varchar(255) NOT NULL, + operation_type varchar(255) NOT NULL, + description varchar(255) NOT NULL, + latency int(11) NOT NULL, + detail varchar(255) DEFAULT NULL, time timestamp NULL DEFAULT CURRENT_TIMESTAMP, - resource_id int(11) NOT NULL, PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 57401291ed..f3d01c14b5 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -2009,10 +2009,14 @@ DROP TABLE IF EXISTS `t_ds_audit_log`; CREATE TABLE `t_ds_audit_log` ( `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT'key', `user_id` int(11) NOT NULL COMMENT 'user id', - `resource_type` int(11) NOT NULL COMMENT 'resource type', - `operation` int(11) NOT NULL COMMENT 'operation', - `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', - `resource_id` int(11) NULL DEFAULT NULL COMMENT 'resource id', + `model_id` bigint(20) DEFAULT NULL COMMENT 'model id', + `model_name` varchar(100) DEFAULT NULL COMMENT 'model name', + `model_type` varchar(100) NOT NULL COMMENT 'model type', + `operation_type` varchar(100) NOT NULL COMMENT 'operation type', + `description` varchar(100) DEFAULT NULL COMMENT 'api description', + `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', + `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', + `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT= 1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 8313542ab8..5dd804c0c7 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -1993,11 +1993,15 @@ CREATE TABLE t_ds_task_group ( DROP TABLE IF EXISTS t_ds_audit_log; CREATE TABLE t_ds_audit_log ( id serial NOT NULL, - user_id int NOT NULL, - resource_type int NOT NULL, - operation int NOT NULL, - time timestamp DEFAULT NULL , - resource_id int NOT NULL, + user_id int NOT NULL, + model_id bigint NOT NULL, + model_name VARCHAR(255) NOT NULL, + model_type VARCHAR(255) NOT NULL, + operation_type VARCHAR(255) NOT NULL, + description VARCHAR(255) NOT NULL, + latency int NOT NULL, + detail VARCHAR(255) DEFAULT NULL, + time timestamp DEFAULT NULL , PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index 5ee7453d46..b5e5b31d11 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -25,4 +25,32 @@ CREATE TABLE `t_ds_relation_project_worker_group` ( UNIQUE KEY unique_project_worker_group(project_code,worker_group) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; -ALTER TABLE t_ds_project_parameter ADD `operator` int(11) DEFAULT NULL COMMENT 'operator user id'; \ No newline at end of file +ALTER TABLE t_ds_project_parameter ADD `operator` int(11) DEFAULT NULL COMMENT 'operator user id'; + +-- modify_data_t_ds_audit_log_input_entry behavior change +--DROP PROCEDURE if EXISTS modify_data_t_ds_audit_log_input_entry; +DROP PROCEDURE if EXISTS modify_data_t_ds_audit_log_input_entry; +delimiter d// +CREATE PROCEDURE modify_data_t_ds_audit_log_input_entry() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_audit_log' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='resource_type') + THEN +ALTER TABLE `t_ds_audit_log` +drop resource_type, drop operation, drop resource_id, + add `model_id` bigint(20) DEFAULT NULL COMMENT 'model id', + add `model_name` varchar(100) DEFAULT NULL COMMENT 'model name', + add `model_type` varchar(100) NOT NULL COMMENT 'model type', + add `operation_type` varchar(100) NOT NULL COMMENT 'operation type', + add `description` varchar(100) DEFAULT NULL COMMENT 'api description', + add `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', + add `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', + MODIFY COLUMN `time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; +END IF; +END; +d// +delimiter ; +CALL modify_data_t_ds_audit_log_input_entry; +DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 90646557c0..22e1c599de 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -29,4 +29,30 @@ DROP SEQUENCE IF EXISTS t_ds_relation_project_worker_group_sequence; CREATE SEQUENCE t_ds_relation_project_worker_group_sequence; ALTER TABLE t_ds_relation_project_worker_group ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_project_worker_group_sequence'); -ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS operator int; \ No newline at end of file +ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS operator int; + +-- modify_data_t_ds_audit_log_input_entry +delimiter d// +CREATE OR REPLACE FUNCTION modify_data_t_ds_audit_log_input_entry() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 't_ds_audit_log' + AND column_name = 'resource_type') + THEN +ALTER TABLE t_ds_audit_log +drop resource_type, drop operation, drop resource_id, + add model_id bigint NOT NULL, + add model_name VARCHAR(255) NOT NULL, + add model_type VARCHAR(255) NOT NULL, + add operation_type VARCHAR(255) NOT NULL, + add description VARCHAR(255) NOT NULL, + add latency int NOT NULL, + add detail VARCHAR(255) DEFAULT NULL; +END IF; +END; +$$ LANGUAGE plpgsql; +d// + +select modify_data_t_ds_audit_log_input_entry(); +DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index cc0532032d..023b559e79 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -17,12 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; +import org.apache.dolphinscheduler.common.enums.AuditModelType; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.AuditLog; import org.apache.dolphinscheduler.dao.entity.Project; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,6 +33,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.Lists; public class AuditLogMapperTest extends BaseDaoTest { @@ -39,13 +43,17 @@ public class AuditLogMapperTest extends BaseDaoTest { @Autowired private ProjectMapper projectMapper; - private void insertOne(AuditResourceType resourceType) { + private void insertOne(AuditModelType objectType) { AuditLog auditLog = new AuditLog(); auditLog.setUserId(1); + auditLog.setModelName("name"); + auditLog.setDetail("detail"); + auditLog.setLatency(1L); auditLog.setTime(new Date()); - auditLog.setResourceType(resourceType.getCode()); - auditLog.setOperation(0); - auditLog.setResourceId(0); + auditLog.setModelType(objectType.getName()); + auditLog.setOperationType(AuditOperationType.CREATE.getName()); + auditLog.setModelId(1L); + auditLog.setDescription("description"); logMapper.insert(auditLog); } @@ -65,25 +73,14 @@ public class AuditLogMapperTest extends BaseDaoTest { */ @Test public void testQueryAuditLog() { - insertOne(AuditResourceType.USER_MODULE); - insertOne(AuditResourceType.PROJECT_MODULE); + insertOne(AuditModelType.USER); + insertOne(AuditModelType.PROJECT); Page page = new Page<>(1, 3); - int[] resourceType = new int[0]; - int[] operationType = new int[0]; + List objectTypeList = new ArrayList<>(); + List operationTypeList = Lists.newArrayList(AuditOperationType.CREATE.getName()); - IPage logIPage = logMapper.queryAuditLog(page, resourceType, operationType, "", null, null); + IPage logIPage = + logMapper.queryAuditLog(page, objectTypeList, operationTypeList, "", "", null, null); Assertions.assertNotEquals(0, logIPage.getTotal()); } - - @Test - public void testQueryResourceNameByType() { - String resourceNameByUser = logMapper.queryResourceNameByType(AuditResourceType.USER_MODULE.getMsg(), 1); - Assertions.assertEquals("admin", resourceNameByUser); - Project project = insertProject(); - String resourceNameByProject = - logMapper.queryResourceNameByType(AuditResourceType.PROJECT_MODULE.getMsg(), project.getId()); - Assertions.assertEquals(project.getName(), resourceNameByProject); - int delete = projectMapper.deleteById(project.getId()); - Assertions.assertEquals(delete, 1); - } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 2b3dbbebbb..40191f6aa3 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -1774,6 +1774,7 @@ public class ProcessServiceImpl implements ProcessService { if (Boolean.TRUE.equals(syncDefine)) { if (processDefinition.getId() == null) { result = processDefineMapper.insert(processDefinitionLog); + processDefinition.setId(processDefinitionLog.getId()); } else { processDefinitionLog.setId(processDefinition.getId()); result = processDefineMapper.updateById(processDefinitionLog); diff --git a/dolphinscheduler-ui/src/locales/en_US/monitor.ts b/dolphinscheduler-ui/src/locales/en_US/monitor.ts index 27bd82ce70..9a7550177a 100644 --- a/dolphinscheduler-ui/src/locales/en_US/monitor.ts +++ b/dolphinscheduler-ui/src/locales/en_US/monitor.ts @@ -60,9 +60,11 @@ export default { }, audit_log: { user_name: 'User Name', - resource_type: 'Resource Type', - project_name: 'Project Name', operation_type: 'Operation Type', + model_type: 'Model Type', + model_name: 'Model Name', + latency: 'Latency', + description: 'Description', create_time: 'Create Time', start_time: 'Start Time', end_time: 'End Time', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts index eca2fe7a20..dbae52fe07 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts @@ -58,9 +58,11 @@ export default { }, audit_log: { user_name: '用户名称', - resource_type: '资源类型', - project_name: '项目名称', operation_type: '操作类型', + model_type: '模型类型', + model_name: '模型名称', + latency: '耗时', + description: '描述', create_time: '创建时间', start_time: '开始时间', end_time: '结束时间', diff --git a/dolphinscheduler-ui/src/service/modules/audit/index.ts b/dolphinscheduler-ui/src/service/modules/audit/index.ts index fd436ccbb2..a436fcdbaf 100644 --- a/dolphinscheduler-ui/src/service/modules/audit/index.ts +++ b/dolphinscheduler-ui/src/service/modules/audit/index.ts @@ -25,3 +25,17 @@ export function queryAuditLogListPaging(params: AuditListReq): any { params }) } + +export function queryAuditModelType(): any { + return axios({ + url: '/projects/audit/audit-log-model-type', + method: 'get' + }) +} + +export function queryAuditLogOperationType(): any { + return axios({ + url: '/projects/audit/audit-log-operation-type', + method: 'get' + }) +} diff --git a/dolphinscheduler-ui/src/service/modules/audit/types.ts b/dolphinscheduler-ui/src/service/modules/audit/types.ts index 030cfc8037..e25efada31 100644 --- a/dolphinscheduler-ui/src/service/modules/audit/types.ts +++ b/dolphinscheduler-ui/src/service/modules/audit/types.ts @@ -45,4 +45,20 @@ interface AuditListRes { start: number } -export { AuditListReq, AuditListRes } +interface AuditModelTypeItem { + code: number + name: string + child: AuditModelTypeItem[] | null +} + +interface AuditOperationTypeItem { + code: number + name: string +} + +export { + AuditListReq, + AuditListRes, + AuditModelTypeItem, + AuditOperationTypeItem +} diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx index 64ae7beff0..17de81fb51 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx @@ -30,7 +30,8 @@ import { NButton, NIcon, NDataTable, - NPagination + NPagination, + NCascader } from 'naive-ui' import { SearchOutlined } from '@vicons/antd' import { useTable } from './use-table' @@ -40,15 +41,23 @@ import Card from '@/components/card' const AuditLog = defineComponent({ name: 'audit-log', setup() { - const { t, variables, getTableData, createColumns } = useTable() + const { + t, + variables, + getTableData, + createColumns, + getModelTypeData, + getOperationTypeData + } = useTable() const requestTableData = () => { getTableData({ pageSize: variables.pageSize, pageNo: variables.page, - resourceType: variables.resourceType, + modelType: variables.modelType, operationType: variables.operationType, userName: variables.userName, + modelName: variables.modelName, datePickerRange: variables.datePickerRange }) } @@ -67,6 +76,8 @@ const AuditLog = defineComponent({ onMounted(() => { createColumns(variables) + getModelTypeData() + getOperationTypeData() requestTableData() }) @@ -97,36 +108,41 @@ const AuditLog = defineComponent({ placeholder={t('monitor.audit_log.user_name')} clearable /> + + - + diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts index 87e5215030..e0a17bbc49 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts @@ -18,22 +18,40 @@ import { useI18n } from 'vue-i18n' import { reactive, ref } from 'vue' import { useAsyncState } from '@vueuse/core' -import { queryAuditLogListPaging } from '@/service/modules/audit' +import { + queryAuditLogListPaging, + queryAuditModelType, + queryAuditLogOperationType +} from '@/service/modules/audit' import { format } from 'date-fns' import { parseTime } from '@/common/common' -import type { AuditListRes } from '@/service/modules/audit/types' +import type { + AuditListRes, + AuditModelTypeItem, + AuditOperationTypeItem +} from '@/service/modules/audit/types' +import { + COLUMN_WIDTH_CONFIG, + calculateTableWidth, + DefaultTableWidth +} from '@/common/column-width-config' +import { sortBy } from 'lodash' export function useTable() { const { t } = useI18n() const variables = reactive({ columns: [], + tableWidth: DefaultTableWidth, tableData: [], page: ref(1), pageSize: ref(10), - resourceType: ref(null), + modelType: ref(null), operationType: ref(null), + ModelTypeData: [], + OperationTypeData: [], userName: ref(null), + modelName: ref(null), datePickerRange: ref(null), totalPage: ref(1), loadingRef: ref(false) @@ -44,29 +62,70 @@ export function useTable() { { title: '#', key: 'index', - render: (row: any, index: number) => index + 1 + render: (row: any, index: number) => index + 1, + ...COLUMN_WIDTH_CONFIG['index'] }, { title: t('monitor.audit_log.user_name'), - key: 'userName' + key: 'userName', + ...COLUMN_WIDTH_CONFIG['userName'] }, { - title: t('monitor.audit_log.resource_type'), - key: 'resource' + title: t('monitor.audit_log.model_type'), + key: 'modelType', + ...COLUMN_WIDTH_CONFIG['type'] }, { - title: t('monitor.audit_log.project_name'), - key: 'resourceName' + title: t('monitor.audit_log.model_name'), + key: 'modelName', + ...COLUMN_WIDTH_CONFIG['name'] }, { title: t('monitor.audit_log.operation_type'), - key: 'operation' + key: 'operation', + ...COLUMN_WIDTH_CONFIG['type'] + }, + + { + title: t('monitor.audit_log.description'), + key: 'description', + ...COLUMN_WIDTH_CONFIG['note'] + }, + { + title: t('monitor.audit_log.latency') + ' (ms)', + key: 'latency', + ...COLUMN_WIDTH_CONFIG['times'] }, { title: t('monitor.audit_log.create_time'), - key: 'time' + key: 'time', + ...COLUMN_WIDTH_CONFIG['time'] } ] + + if (variables.tableWidth) { + variables.tableWidth = calculateTableWidth(variables.columns) + } + } + + const getModelTypeData = async () => { + try { + variables.ModelTypeData = await queryAuditModelType().then( + (res: AuditModelTypeItem[]) => res || [] + ) + } catch { + variables.ModelTypeData = [] + } + } + + const getOperationTypeData = async () => { + try { + variables.OperationTypeData = await queryAuditLogOperationType().then( + (res: AuditOperationTypeItem[]) => sortBy(res, 'name') + ) + } catch { + variables.OperationTypeData = [] + } } const getTableData = (params: any) => { @@ -75,9 +134,10 @@ export function useTable() { const data = { pageSize: params.pageSize, pageNo: params.pageNo, - resourceType: params.resourceType, - operationType: params.operationType, + modelTypes: params.modelType, + operationTypes: params.operationType, userName: params.userName, + modelName: params.modelName, startDate: params.datePickerRange ? format(parseTime(params.datePickerRange[0]), 'yyyy-MM-dd HH:mm:ss') : '', @@ -106,6 +166,8 @@ export function useTable() { t, variables, getTableData, - createColumns + createColumns, + getModelTypeData: getModelTypeData, + getOperationTypeData } } From 7894ebb3506d4281c27715f684e9fb5701bf5351 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 15 Apr 2024 14:15:44 +0800 Subject: [PATCH 47/88] [TEST] increase coverage of environment service (#15840) * [TEST] increase coverage of environment service * spotless apply --- .../service/impl/EnvironmentServiceTest.java | 88 ++++++++++++++++++- .../api/utils/ServiceTestUtil.java | 30 +++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java index 14e1ed956b..b226295e0e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServi import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_UPDATE; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -29,9 +30,11 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.ServiceTestUtil; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.User; @@ -42,6 +45,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -53,6 +57,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -91,6 +96,9 @@ public class EnvironmentServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; + @Mock + private CodeGenerateUtils codeGenerateUtils; + public static final String testUserName = "environmentServerTest"; public static final String environmentName = "Env1"; @@ -121,9 +129,24 @@ public class EnvironmentServiceTest { when(environmentMapper.insert(any(Environment.class))).thenReturn(1); when(relationMapper.insert(any(EnvironmentWorkerGroupRelation.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); assertDoesNotThrow( () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + when(environmentMapper.insert(any(Environment.class))).thenReturn(-1); + assertThrowsServiceException(Status.CREATE_ENVIRONMENT_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.getInstance()).thenReturn(codeGenerateUtils); + when(codeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + } } @Test @@ -156,25 +179,54 @@ public class EnvironmentServiceTest { assertThrowsServiceException(Status.ENVIRONMENT_NAME_EXISTS, () -> environmentService .updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), getDesc(), workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(-1); + assertThrowsServiceException(Status.UPDATE_ENVIRONMENT_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", workerGroups)); + + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); + + when(relationMapper.queryByEnvironmentCode(any())) + .thenReturn(Collections.singletonList(getEnvironmentWorkerGroup())); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); } @Test public void testQueryAllEnvironmentList() { + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, + 1, environmentServiceLogger)).thenReturn(Collections.emptySet()); + Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + assertEquals(0, ((List) result.get(Constants.DATA_LIST)).size()); + Set ids = new HashSet<>(); ids.add(1); when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, 1, environmentServiceLogger)).thenReturn(ids); when(environmentMapper.selectBatchIds(ids)).thenReturn(Lists.newArrayList(getEnvironment())); - Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List list = (List) (result.get(Constants.DATA_LIST)); Assertions.assertEquals(1, list.size()); + + when(environmentMapper.selectBatchIds(ids)).thenReturn(Collections.emptyList()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); + Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + list = (List) (result.get(Constants.DATA_LIST)); + Assertions.assertEquals(0, list.size()); } @Test @@ -186,9 +238,27 @@ public class EnvironmentServiceTest { .thenReturn(page); Result result = environmentService.queryEnvironmentListPaging(getAdminUser(), 1, 10, environmentName); - logger.info(result.toString()); PageInfo pageInfo = (PageInfo) result.getData(); Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); + + assertDoesNotThrow( + () -> environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName)); + + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition( + AuthorizationType.ENVIRONMENT, + 1, + environmentServiceLogger)).thenReturn(Collections.singleton(10)); + when(environmentMapper.queryEnvironmentListPagingByIds(any(Page.class), any(List.class), any(String.class))) + .thenReturn(page); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(1, ((PageInfo) result.getData()).getTotalList().size()); + + page.setRecords(Collections.emptyList()); + page.setTotal(0); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(0, ((PageInfo) result.getData()).getTotalList().size()); } @Test @@ -239,6 +309,10 @@ public class EnvironmentServiceTest { result = environmentService.deleteEnvironmentByCode(loginUser, 1L); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + when(environmentMapper.deleteByCode(1L)).thenReturn(-1); + result = environmentService.deleteEnvironmentByCode(loginUser, 1L); + Assertions.assertEquals(Status.DELETE_ENVIRONMENT_ERROR, result.get(Constants.STATUS)); } @Test @@ -251,6 +325,9 @@ public class EnvironmentServiceTest { result = environmentService.verifyEnvironment(environmentName); logger.info(result.toString()); Assertions.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + + when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(null); + assertDoesNotThrow(() -> environmentService.verifyEnvironment(environmentName)); } private Environment getEnvironment() { @@ -264,6 +341,13 @@ public class EnvironmentServiceTest { return environment; } + private EnvironmentWorkerGroupRelation getEnvironmentWorkerGroup() { + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(1L); + relation.setWorkerGroup("new_worker_group"); + return relation; + } + /** * create an environment description */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java new file mode 100644 index 0000000000..0dc71841b4 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import java.nio.charset.StandardCharsets; +import java.util.Random; + +public class ServiceTestUtil { + + public static String randomStringWithLengthN(int n) { + byte[] bitArray = new byte[n]; + new Random().nextBytes(bitArray); + return new String(bitArray, StandardCharsets.UTF_8); + } +} From 5ad7b1509f79c52b24d11f50c1d0172287dcd968 Mon Sep 17 00:00:00 2001 From: XinXing <49302071+xinxingi@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:01:28 +0800 Subject: [PATCH 48/88] =?UTF-8?q?[Fix-15787]=20Reuse=20code=20and=20solve?= =?UTF-8?q?=20the=20problem=20of=20complex=20SQL=20parsing=20exceptions=20?= =?UTF-8?q?in=E2=80=A6=20(#15833)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reuse code and solve the problem of complex SQL parsing exceptions in druid, corresponding to issue #15787 * Code Format * Enhanced adaptability to SQL formatting --- .../oracle/param/OracleDataSourceProcessor.java | 11 ++++++----- .../oracle/param/OracleDataSourceProcessorTest.java | 13 ++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java index 89b872d7f5..5f24b8cc19 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java @@ -41,8 +41,7 @@ import java.util.stream.Collectors; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; -import com.alibaba.druid.sql.parser.SQLParserFeature; -import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.alibaba.druid.sql.parser.SQLParserUtils; import com.google.auto.service.AutoService; @AutoService(DataSourceProcessor.class) @@ -149,9 +148,11 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - SQLStatementParser parser = new OracleStatementParser(sql, SQLParserFeature.KeepComments); - List statementList = parser.parseStatementList(); - return statementList.stream().map(SQLStatement::toString).collect(Collectors.toList()); + if (sql.toUpperCase().contains("BEGIN") && sql.toUpperCase().contains("END")) { + return new OracleStatementParser(sql).parseStatementList().stream().map(SQLStatement::toString) + .collect(Collectors.toList()); + } + return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.oracle); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java index 57e316ed63..2f9133463d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java @@ -155,13 +155,16 @@ public class OracleDataSourceProcessorTest { @Test void splitAndRemoveComment_MultipleSql() { - String plSql = "select * from test;select * from test2;"; + String plSql = + "select a,a-a as b from (select 1 as a,2 as b from dual) union all select 1 as a,2 as b from dual;select * from dual; -- this comment"; List sqls = oracleDatasourceProcessor.splitAndRemoveComment(plSql); // We will not split the plsql Assertions.assertEquals(2, sqls.size()); - Assertions.assertEquals("SELECT *\n" + - "FROM test;", sqls.get(0)); - Assertions.assertEquals("SELECT *\n" + - "FROM test2;", sqls.get(1)); + System.out.println(sqls.get(0)); + System.out.println(sqls.get(1)); + Assertions.assertEquals( + "select a,a-a as b from (select 1 as a,2 as b from dual) union all select 1 as a,2 as b from dual", + sqls.get(0)); + Assertions.assertEquals("select * from dual", sqls.get(1)); } } From 6844c659bf44f7f027adbe7a64723a1ca0f1cd89 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 15 Apr 2024 16:14:32 +0800 Subject: [PATCH 49/88] Fix ErrorCommand loss some fields in Command (#15847) --- .../dao/entity/ErrorCommand.java | 17 +++- .../dao/entity/ErrorCommandTest.java | 82 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index aa4d3d4782..d52984b61f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -52,6 +52,10 @@ public class ErrorCommand { */ private long processDefinitionCode; + private int processDefinitionVersion; + + private int processInstanceId; + /** * executor id */ @@ -73,7 +77,7 @@ public class ErrorCommand { private FailureStrategy failureStrategy; /** - * warning type + * warning type */ private WarningType warningType; @@ -135,21 +139,26 @@ public class ErrorCommand { public ErrorCommand() { } + public ErrorCommand(Command command, String message) { this.id = command.getId(); this.commandType = command.getCommandType(); this.executorId = command.getExecutorId(); this.processDefinitionCode = command.getProcessDefinitionCode(); + this.processDefinitionVersion = command.getProcessDefinitionVersion(); + this.processInstanceId = command.getProcessInstanceId(); this.commandParam = command.getCommandParam(); + this.taskDependType = command.getTaskDependType(); + this.failureStrategy = command.getFailureStrategy(); this.warningType = command.getWarningType(); this.warningGroupId = command.getWarningGroupId(); this.scheduleTime = command.getScheduleTime(); - this.taskDependType = command.getTaskDependType(); - this.failureStrategy = command.getFailureStrategy(); this.startTime = command.getStartTime(); this.updateTime = command.getUpdateTime(); - this.environmentCode = command.getEnvironmentCode(); this.processInstancePriority = command.getProcessInstancePriority(); + this.workerGroup = command.getWorkerGroup(); + this.tenantCode = command.getTenantCode(); + this.environmentCode = command.getEnvironmentCode(); this.message = message; this.dryRun = command.getDryRun(); this.testFlag = command.getTestFlag(); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java new file mode 100644 index 0000000000..057d6578e6 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ErrorCommandTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; + +import java.util.Date; + +import org.junit.jupiter.api.Test; + +class ErrorCommandTest { + + @Test + void testConstructor() { + Command command = new Command(); + command.setId(1); + command.setCommandType(CommandType.PAUSE); + command.setExecutorId(1); + command.setProcessDefinitionCode(123); + command.setProcessDefinitionVersion(1); + command.setProcessInstanceId(1); + command.setCommandParam("param"); + command.setTaskDependType(TaskDependType.TASK_POST); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(new Date()); + command.setStartTime(new Date()); + command.setUpdateTime(new Date()); + command.setProcessInstancePriority(Priority.HIGHEST); + command.setWorkerGroup("default"); + command.setTenantCode("root"); + command.setEnvironmentCode(1L); + command.setDryRun(1); + command.setTestFlag(Flag.NO.getCode()); + + ErrorCommand errorCommand = new ErrorCommand(command, "test"); + assertEquals(command.getCommandType(), errorCommand.getCommandType()); + assertEquals(command.getExecutorId(), errorCommand.getExecutorId()); + assertEquals(command.getProcessDefinitionCode(), errorCommand.getProcessDefinitionCode()); + assertEquals(command.getProcessDefinitionVersion(), errorCommand.getProcessDefinitionVersion()); + assertEquals(command.getProcessInstanceId(), errorCommand.getProcessInstanceId()); + assertEquals(command.getCommandParam(), errorCommand.getCommandParam()); + assertEquals(command.getTaskDependType(), errorCommand.getTaskDependType()); + assertEquals(command.getFailureStrategy(), errorCommand.getFailureStrategy()); + assertEquals(command.getWarningType(), errorCommand.getWarningType()); + assertEquals(command.getWarningGroupId(), errorCommand.getWarningGroupId()); + assertEquals(command.getScheduleTime(), errorCommand.getScheduleTime()); + assertEquals(command.getStartTime(), errorCommand.getStartTime()); + assertEquals(command.getUpdateTime(), errorCommand.getUpdateTime()); + assertEquals(command.getProcessInstancePriority(), errorCommand.getProcessInstancePriority()); + assertEquals(command.getWorkerGroup(), errorCommand.getWorkerGroup()); + assertEquals(command.getTenantCode(), errorCommand.getTenantCode()); + assertEquals(command.getEnvironmentCode(), errorCommand.getEnvironmentCode()); + assertEquals(command.getDryRun(), errorCommand.getDryRun()); + assertEquals(command.getTestFlag(), errorCommand.getTestFlag()); + assertEquals("test", errorCommand.getMessage()); + } + +} From f99d3f1ed37f1f36a51cfa2063d298f1bed5de49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Mon, 15 Apr 2024 17:01:06 +0800 Subject: [PATCH 50/88] [Improvement][Audit] Change time to create_time (#15846) * change time to create_time * update * update --- .../api/audit/OperatorUtils.java | 2 +- .../dolphinscheduler/api/dto/AuditDto.java | 2 +- .../api/service/impl/AuditServiceImpl.java | 2 +- .../dolphinscheduler/dao/entity/AuditLog.java | 2 +- .../dao/mapper/AuditLogMapper.xml | 8 ++++---- .../resources/sql/dolphinscheduler_h2.sql | 2 +- .../resources/sql/dolphinscheduler_mysql.sql | 2 +- .../sql/dolphinscheduler_postgresql.sql | 2 +- .../mysql/dolphinscheduler_ddl.sql | 2 +- .../postgresql/dolphinscheduler_ddl.sql | 19 ++++++++++--------- .../dao/mapper/AuditLogMapperTest.java | 2 +- .../monitor/statistics/audit-log/use-table.ts | 2 +- 12 files changed, 24 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index e8b6b1254c..a0233c7e93 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -65,7 +65,7 @@ public class OperatorUtils { auditLog.setModelType(auditType.getAuditModelType().getName()); auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); - auditLog.setTime(new Date()); + auditLog.setCreateTime(new Date()); auditLogList.add(auditLog); return auditLogList; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java index 2c4144220b..0b36325b44 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java @@ -34,7 +34,7 @@ public class AuditDto { private String operation; - private Date time; + private Date createTime; private String description; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index bdc7a4c573..2ec547bcb6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -129,7 +129,7 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { auditDto.setLatency(String.valueOf(auditLog.getLatency())); auditDto.setDetail(auditLog.getDetail()); auditDto.setDescription(auditLog.getDescription()); - auditDto.setTime(auditLog.getTime()); + auditDto.setCreateTime(auditLog.getCreateTime()); return auditDto; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java index 397fa722e5..7c6f76e1da 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java @@ -67,7 +67,7 @@ public class AuditLog { /** * operation time */ - private Date time; + private Date createTime; private String detail; diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml index 94430a343a..63adb57c34 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml @@ -19,10 +19,10 @@ - id, user_id, model_type, operation_type, model_id, model_name, time, detail, description, latency + id, user_id, model_type, operation_type, model_id, model_name, create_time, detail, description, latency - ${alias}.id, ${alias}.user_id, ${alias}.model_type, ${alias}.operation_type, ${alias}.model_id, ${alias}.model_name, ${alias}.time, ${alias}.detail, ${alias}.description, ${alias}.latency + ${alias}.id, ${alias}.user_id, ${alias}.model_type, ${alias}.operation_type, ${alias}.model_id, ${alias}.model_name, ${alias}.create_time, ${alias}.detail, ${alias}.description, ${alias}.latency diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 4187b93a1c..42c893bb05 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -2027,7 +2027,7 @@ CREATE TABLE t_ds_audit_log description varchar(255) NOT NULL, latency int(11) NOT NULL, detail varchar(255) DEFAULT NULL, - time timestamp NULL DEFAULT CURRENT_TIMESTAMP, + create_time timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index f3d01c14b5..e2138b67a2 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -2016,7 +2016,7 @@ CREATE TABLE `t_ds_audit_log` ( `description` varchar(100) DEFAULT NULL COMMENT 'api description', `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', - `time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'operation time', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT= 1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 5dd804c0c7..2d224092c4 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -2001,7 +2001,7 @@ CREATE TABLE t_ds_audit_log ( description VARCHAR(255) NOT NULL, latency int NOT NULL, detail VARCHAR(255) DEFAULT NULL, - time timestamp DEFAULT NULL , + create_time timestamp DEFAULT NULL , PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index b5e5b31d11..d10ac6b710 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -47,7 +47,7 @@ drop resource_type, drop operation, drop resource_id, add `description` varchar(100) DEFAULT NULL COMMENT 'api description', add `latency` int(11) DEFAULT NULL COMMENT 'api cost milliseconds', add `detail` varchar(100) DEFAULT NULL COMMENT 'object change detail', - MODIFY COLUMN `time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; + CHANGE COLUMN `time` `create_time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "operation time"; END IF; END; d// diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 22e1c599de..ee06588e58 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -40,15 +40,16 @@ BEGIN WHERE table_name = 't_ds_audit_log' AND column_name = 'resource_type') THEN -ALTER TABLE t_ds_audit_log -drop resource_type, drop operation, drop resource_id, - add model_id bigint NOT NULL, - add model_name VARCHAR(255) NOT NULL, - add model_type VARCHAR(255) NOT NULL, - add operation_type VARCHAR(255) NOT NULL, - add description VARCHAR(255) NOT NULL, - add latency int NOT NULL, - add detail VARCHAR(255) DEFAULT NULL; + ALTER TABLE t_ds_audit_log + drop resource_type, drop operation, drop resource_id, + add model_id bigint NOT NULL, + add model_name VARCHAR(255) NOT NULL, + add model_type VARCHAR(255) NOT NULL, + add operation_type VARCHAR(255) NOT NULL, + add description VARCHAR(255) NOT NULL, + add latency int NOT NULL, + add detail VARCHAR(255) DEFAULT NULL; + ALTER TABLE t_ds_audit_log RENAME COLUMN "time" TO "create_time"; END IF; END; $$ LANGUAGE plpgsql; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index 023b559e79..92418193b8 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -49,7 +49,7 @@ public class AuditLogMapperTest extends BaseDaoTest { auditLog.setModelName("name"); auditLog.setDetail("detail"); auditLog.setLatency(1L); - auditLog.setTime(new Date()); + auditLog.setCreateTime(new Date()); auditLog.setModelType(objectType.getName()); auditLog.setOperationType(AuditOperationType.CREATE.getName()); auditLog.setModelId(1L); diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts index e0a17bbc49..385299d019 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/use-table.ts @@ -98,7 +98,7 @@ export function useTable() { }, { title: t('monitor.audit_log.create_time'), - key: 'time', + key: 'createTime', ...COLUMN_WIDTH_CONFIG['time'] } ] From 27d0563fe4c020a771448830309429592c66ba9c Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 10:58:43 +0800 Subject: [PATCH 51/88] Bind processId to constructor CodeGenerator (#15848) --- .../api/python/PythonGateway.java | 6 +- .../api/service/impl/ClusterServiceImpl.java | 2 +- .../service/impl/EnvironmentServiceImpl.java | 2 +- .../api/service/impl/ExecutorServiceImpl.java | 2 +- .../service/impl/K8SNamespaceServiceImpl.java | 2 +- .../impl/ProcessDefinitionServiceImpl.java | 16 ++-- .../impl/ProjectParameterServiceImpl.java | 2 +- .../impl/ProjectPreferenceServiceImpl.java | 2 +- .../api/service/impl/ProjectServiceImpl.java | 2 +- .../impl/TaskDefinitionServiceImpl.java | 6 +- .../service/impl/EnvironmentServiceTest.java | 6 +- .../common/utils/CodeGenerateUtils.java | 95 ++++++++++--------- .../common/utils/CodeGenerateUtilsTest.java | 55 +++++++++-- .../service/process/ProcessServiceImpl.java | 2 +- .../datasource/dao/ProcessDefinitionDao.java | 2 +- .../tools/datasource/dao/ProjectDao.java | 2 +- .../v200/V200DolphinSchedulerUpgrader.java | 2 +- .../tools/demo/ProcessDefinitionDemo.java | 16 ++-- 18 files changed, 132 insertions(+), 90 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 1e4e1f5aa0..762ba576b0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -184,7 +184,7 @@ public class PythonGateway { Map result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value if (project == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -194,7 +194,7 @@ public class PythonGateway { // In the case project exists, but current workflow still not created, we should also return the init // version of it if (processDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -202,7 +202,7 @@ public class PythonGateway { TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); if (taskDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); } else { result.put("code", taskDefinition.getCode()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index a5ceb92abc..bf4e3ac4d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -96,7 +96,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic cluster.setOperator(loginUser.getId()); cluster.setCreateTime(new Date()); cluster.setUpdateTime(new Date()); - cluster.setCode(CodeGenerateUtils.getInstance().genCode()); + cluster.setCode(CodeGenerateUtils.genCode()); if (clusterMapper.insert(cluster) > 0) { return cluster.getCode(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index ade9aea483..2eb8f84810 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -123,7 +123,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme env.setUpdateTime(new Date()); long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); env.setCode(code); } catch (CodeGenerateException e) { log.error("Generate environment code error.", e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 7ab6102ff8..5b576ce95a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -258,7 +258,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ checkScheduleTimeNumExceed(commandType, cronTime); checkMasterExists(); - long triggerCode = CodeGenerateUtils.getInstance().genCode(); + long triggerCode = CodeGenerateUtils.genCode(); /** * create command diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java index 54894a5ca7..7543616f31 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java @@ -141,7 +141,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); cluster.setCode(code); } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("Generate cluster code error.", e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index ed6b2d27bb..2bf84a0bcc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -299,7 +299,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro List taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson); List taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs); - long processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + long processDefinitionCode = CodeGenerateUtils.genCode(); ProcessDefinition processDefinition = new ProcessDefinition(projectCode, name, processDefinitionCode, description, globalParams, locations, timeout, loginUser.getId()); @@ -360,7 +360,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro long processDefinitionCode; try { - processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + processDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -1233,7 +1233,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // build process definition processDefinition = new ProcessDefinition(projectCode, processDefinitionName, - CodeGenerateUtils.getInstance().genCode(), + CodeGenerateUtils.genCode(), "", "[]", null, 0, loginUser.getId()); @@ -1388,7 +1388,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro sqlParameters.setSqlType(SqlType.NON_QUERY.ordinal()); sqlParameters.setLocalParams(Collections.emptyList()); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); taskDefinition.setTaskType(TASK_TYPE_SQL); taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryInterval(0); @@ -1433,7 +1433,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processDefinition.setProjectCode(projectCode); processDefinition.setUserId(loginUser.getId()); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error( "Save process definition error because generate process definition code error, projectCode:{}.", @@ -1456,7 +1456,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperateTime(now); try { - long code = CodeGenerateUtils.getInstance().genCode(); + long code = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), code); taskDefinitionLog.setCode(code); } catch (CodeGenerateException e) { @@ -2074,7 +2074,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro Map taskCodeMap = new HashMap<>(); for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { try { - long taskCode = CodeGenerateUtils.getInstance().genCode(); + long taskCode = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), taskCode); taskDefinitionLog.setCode(taskCode); } catch (CodeGenerateException e) { @@ -2097,7 +2097,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } final long oldProcessDefinitionCode = processDefinition.getCode(); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error("Generate process definition code error, projectCode:{}.", targetProjectCode, e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e30375e809..e0011096e4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -97,7 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj .builder() .paramName(projectParameterName) .paramValue(projectParameterValue) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .projectCode(projectCode) .userId(loginUser.getId()) .createTime(now) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java index dcbd3b7456..6274d290d5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java @@ -76,7 +76,7 @@ public class ProjectPreferenceServiceImpl extends BaseServiceImpl projectPreference.setProjectCode(projectCode); projectPreference.setPreferences(preferences); projectPreference.setUserId(loginUser.getId()); - projectPreference.setCode(CodeGenerateUtils.getInstance().genCode()); + projectPreference.setCode(CodeGenerateUtils.genCode()); projectPreference.setState(1); projectPreference.setCreateTime(now); projectPreference.setUpdateTime(now); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 1a00584a26..b5c329c4fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -126,7 +126,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic project = Project .builder() .name(name) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .description(desc) .userId(loginUser.getId()) .userName(loginUser.getUserName()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 4a0c3f8b68..8b01df1319 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -265,7 +265,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe long taskDefinitionCode; try { - taskDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + taskDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -338,7 +338,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe } long taskCode = taskDefinition.getCode(); if (taskCode == 0) { - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); } List processTaskRelationLogList = processTaskRelationMapper.queryByProcessCode(processDefinitionCode) @@ -1264,7 +1264,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe List taskCodes = new ArrayList<>(); try { for (int i = 0; i < genNum; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateException e) { log.error("Generate task definition code error.", e); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java index b226295e0e..b8161af13d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java @@ -96,9 +96,6 @@ public class EnvironmentServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; - @Mock - private CodeGenerateUtils codeGenerateUtils; - public static final String testUserName = "environmentServerTest"; public static final String environmentName = "Env1"; @@ -141,8 +138,7 @@ public class EnvironmentServiceTest { () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { - when(CodeGenerateUtils.getInstance()).thenReturn(codeGenerateUtils); - when(codeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java index f35523b59d..3e75264808 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java @@ -1,4 +1,6 @@ -/** Copyright 2010-2012 Twitter, Inc.*/ +/** + * Copyright 2010-2012 Twitter, Inc. + */ package org.apache.dolphinscheduler.common.utils; @@ -6,66 +8,71 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; + /** - * Rewriting based on Twitter snowflake algorithm + * Rewriting based on Twitter snowflake algorithm */ +@Slf4j public class CodeGenerateUtils { - // start timestamp - private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 - // Each machine generates 32 in the same millisecond - private static final long LOW_DIGIT_BIT = 5L; - private static final long MIDDLE_BIT = 2L; - private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); - // The displacement to the left - private static final long MIDDLE_LEFT = LOW_DIGIT_BIT; - private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT; - private final long machineHash; - private long lowDigit = 0L; - private long recordMillisecond = -1L; + private static final CodeGenerator codeGenerator; - private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); - private static final long SYSTEM_NANOTIME = System.nanoTime(); - - private CodeGenerateUtils() throws CodeGenerateException { + static { try { - this.machineHash = - Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1)); + codeGenerator = new CodeGenerator(InetAddress.getLocalHost().getHostName() + "-" + OSUtils.getProcessID()); } catch (UnknownHostException e) { throw new CodeGenerateException(e.getMessage()); } } - private static CodeGenerateUtils instance = null; - - public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException { - if (instance == null) { - instance = new CodeGenerateUtils(); - } - return instance; + public static long genCode() throws CodeGenerateException { + return codeGenerator.genCode(); } - public synchronized long genCode() throws CodeGenerateException { - long nowtMillisecond = systemMillisecond(); - if (nowtMillisecond < recordMillisecond) { - throw new CodeGenerateException("New code exception because time is set back."); + public static class CodeGenerator { + + // start timestamp + private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 + // Each machine generates 32 in the same millisecond + private static final long LOW_DIGIT_BIT = 5L; + private static final long MACHINE_BIT = 5L; + private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); + // The displacement to the left + private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MACHINE_BIT; + public final long machineHash; + private long lowDigit = 0L; + private long recordMillisecond = -1L; + + private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); + private static final long SYSTEM_NANOTIME = System.nanoTime(); + + public CodeGenerator(String appName) { + this.machineHash = Math.abs(Objects.hash(appName)) % (1 << MACHINE_BIT); } - if (nowtMillisecond == recordMillisecond) { - lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; - if (lowDigit == 0L) { - while (nowtMillisecond <= recordMillisecond) { - nowtMillisecond = systemMillisecond(); - } + + public synchronized long genCode() throws CodeGenerateException { + long nowtMillisecond = systemMillisecond(); + if (nowtMillisecond < recordMillisecond) { + throw new CodeGenerateException("New code exception because time is set back."); } - } else { - lowDigit = 0L; + if (nowtMillisecond == recordMillisecond) { + lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; + if (lowDigit == 0L) { + while (nowtMillisecond <= recordMillisecond) { + nowtMillisecond = systemMillisecond(); + } + } + } else { + lowDigit = 0L; + } + recordMillisecond = nowtMillisecond; + return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << LOW_DIGIT_BIT | lowDigit; } - recordMillisecond = nowtMillisecond; - return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit; - } - private long systemMillisecond() { - return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + private long systemMillisecond() { + return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + } } public static class CodeGenerateException extends RuntimeException { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index 3871646c95..8cd8ab8e6d 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -17,20 +17,59 @@ package org.apache.dolphinscheduler.common.utils; -import java.util.HashSet; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class CodeGenerateUtilsTest { +class CodeGenerateUtilsTest { @Test - public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { - HashSet existsCode = new HashSet<>(); - for (int i = 0; i < 100; i++) { - Long currentCode = CodeGenerateUtils.getInstance().genCode(); - Assertions.assertFalse(existsCode.contains(currentCode)); + void testNoGenerateDuplicateCode() { + int codeNum = 10000000; + List existsCode = new ArrayList<>(); + for (int i = 0; i < codeNum; i++) { + Long currentCode = CodeGenerateUtils.genCode(); existsCode.add(currentCode); } + Set existsCodeSet = new HashSet<>(existsCode); + // Disallow duplicate code + assertEquals(existsCode.size(), existsCodeSet.size()); + } + + @Test + void testNoGenerateDuplicateCodeWithDifferentAppName() throws UnknownHostException, InterruptedException { + int threadNum = 10; + int codeNum = 1000000; + + final String hostName = InetAddress.getLocalHost().getHostName(); + Map> machineCodes = new ConcurrentHashMap<>(); + CountDownLatch countDownLatch = new CountDownLatch(threadNum); + + for (int i = 0; i < threadNum; i++) { + final int c = i; + new Thread(() -> { + List codes = new ArrayList<>(codeNum); + CodeGenerateUtils.CodeGenerator codeGenerator = new CodeGenerateUtils.CodeGenerator(hostName + "-" + c); + for (int j = 0; j < codeNum; j++) { + codes.add(codeGenerator.genCode()); + } + machineCodes.put(Thread.currentThread().getName(), codes); + countDownLatch.countDown(); + }).start(); + } + countDownLatch.await(); + Set totalCodes = new HashSet<>(); + machineCodes.values().forEach(totalCodes::addAll); + assertEquals(codeNum * threadNum, totalCodes.size()); } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 40191f6aa3..028ab7651f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -1679,7 +1679,7 @@ public class ProcessServiceImpl implements ProcessService { taskDefinitionLog.setOperateTime(now); taskDefinitionLog.setOperator(operator.getId()); if (taskDefinitionLog.getCode() == 0) { - taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinitionLog.setCode(CodeGenerateUtils.genCode()); } if (taskDefinitionLog.getVersion() == 0) { // init first version diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java index 338a9d591d..877f1c9482 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java @@ -100,7 +100,7 @@ public class ProcessDefinitionDao { processDefinition.setId(rs.getInt(1)); long code = rs.getLong(2); if (code == 0L) { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); } processDefinition.setCode(code); processDefinition.setVersion(Constants.VERSION_FIRST); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java index 65466fe99a..685732337a 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java @@ -47,7 +47,7 @@ public class ProjectDao { Integer id = rs.getInt(1); long code = rs.getLong(2); if (code == 0L) { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); } projectMap.put(id, code); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java index 35a9be75dc..fd430c2b06 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java @@ -206,7 +206,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { taskDefinitionLog.setName(name); taskDefinitionLog .setWorkerGroup(task.get("workerGroup") == null ? "default" : task.get("workerGroup").asText()); - long taskCode = CodeGenerateUtils.getInstance().genCode(); + long taskCode = CodeGenerateUtils.genCode(); taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setVersion(Constants.VERSION_FIRST); taskDefinitionLog.setProjectCode(processDefinition.getProjectCode()); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java index fb1cfab637..023928575f 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java @@ -85,7 +85,7 @@ public class ProcessDefinitionDemo { project = Project .builder() .name("demo") - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .description("") .userId(loginUser.getId()) .userName(loginUser.getUserName()) @@ -167,7 +167,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 1; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -242,7 +242,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 2; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -334,7 +334,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 2; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -420,7 +420,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 4; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -537,7 +537,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 4; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -656,7 +656,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 3; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); @@ -755,7 +755,7 @@ public class ProcessDefinitionDemo { List taskCodes = new ArrayList<>(); try { for (int i = 0; i < 1; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("task code get error, ", e); From 76d059810a928c3a122df3ee917cb5eedaf60063 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 16:19:57 +0800 Subject: [PATCH 52/88] Forbidden forcess success a task instance in a running workflow instance (#15855) --- .../controller/TaskInstanceController.java | 9 +- .../v2/TaskInstanceV2Controller.java | 4 +- .../api/service/TaskInstanceService.java | 6 +- .../service/impl/TaskInstanceServiceImpl.java | 64 ++++------ .../api/AssertionsHelper.java | 8 ++ .../TaskInstanceControllerTest.java | 4 +- .../v2/TaskInstanceV2ControllerTest.java | 8 +- .../api/service/TaskInstanceServiceTest.java | 119 +++++++++--------- .../service/process/ProcessService.java | 2 +- .../service/process/ProcessServiceImpl.java | 13 +- 10 files changed, 116 insertions(+), 121 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 1865dfee5d..e0055595c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -153,10 +153,11 @@ public class TaskInstanceController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(FORCE_TASK_SUCCESS_ERROR) @OperatorLog(auditType = AuditType.TASK_INSTANCE_FORCE_SUCCESS) - public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @Schema(name = "projectCode", required = true) @PathVariable long projectCode, - @PathVariable(value = "id") Integer id) { - return taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @Schema(name = "projectCode", required = true) @PathVariable long projectCode, + @PathVariable(value = "id") Integer id) { + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return Result.success(); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java index 3e3a87681b..ea767f0fa3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java @@ -167,8 +167,8 @@ public class TaskInstanceV2Controller extends BaseController { public TaskInstanceSuccessResponse forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "id") Integer id) { - Result result = taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); - return new TaskInstanceSuccessResponse(result); + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return new TaskInstanceSuccessResponse(Result.success()); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 86e5396dbe..bff0518685 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -72,9 +72,9 @@ public interface TaskInstanceService { * @param taskInstanceId task instance id * @return the result code and msg */ - Result forceTaskSuccess(User loginUser, - long projectCode, - Integer taskInstanceId); + void forceTaskSuccess(User loginUser, + long projectCode, + Integer taskInstanceId); /** * task savepoint diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index f06f8115a9..7469d8db13 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -23,6 +23,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.service.TaskGroupQueueService; import org.apache.dolphinscheduler.api.service.TaskInstanceService; @@ -33,14 +34,15 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.repository.DqExecuteResultDao; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils; import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; @@ -107,6 +109,9 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst @Autowired private TaskGroupQueueService taskGroupQueueService; + @Autowired + private ProcessInstanceDao workflowInstanceDao; + /** * query task list by project, process instance, task name, task start time, task end time, task status, keyword paging * @@ -216,58 +221,39 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst */ @Transactional @Override - public Result forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { - Result result = new Result(); - Project project = projectMapper.queryByCode(projectCode); + public void forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { // check user access for project - Map checkResult = - projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); - Status status = (Status) checkResult.get(Constants.STATUS); - if (status != Status.SUCCESS) { - putMsg(result, status); - return result; + projectService.checkProjectAndAuthThrowException(loginUser, projectCode, FORCED_SUCCESS); + + TaskInstance task = taskInstanceDao.queryOptionalById(taskInstanceId) + .orElseThrow(() -> new ServiceException(Status.TASK_INSTANCE_NOT_FOUND)); + + if (task.getProjectCode() != projectCode) { + throw new ServiceException("The task instance is not under the project: " + projectCode); } - // check whether the task instance can be found - TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); - if (task == null) { - log.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, - taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); - return result; - } - - TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(task.getTaskCode()); - if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - log.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, - task.getTaskCode()); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND, taskInstanceId); - return result; + ProcessInstance processInstance = workflowInstanceDao.queryOptionalById(task.getProcessInstanceId()) + .orElseThrow( + () -> new ServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, task.getProcessInstanceId())); + if (!processInstance.getState().isFinished()) { + throw new ServiceException("The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance"); } // check whether the task instance state type is failure or cancel if (!task.getState().isFailure() && !task.getState().isKill()) { - log.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", - task.getState().getDesc(), projectCode, taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState().toString()); - return result; + throw new ServiceException(Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState()); } // change the state of the task instance task.setState(TaskExecutionStatus.FORCED_SUCCESS); task.setEndTime(new Date()); int changedNum = taskInstanceMapper.updateById(task); - if (changedNum > 0) { - processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId); - log.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, - taskInstanceId); - putMsg(result, Status.SUCCESS); - } else { - log.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", - projectCode, taskInstanceId); - putMsg(result, Status.FORCE_TASK_SUCCESS_ERROR); + if (changedNum <= 0) { + throw new ServiceException(Status.FORCE_TASK_SUCCESS_ERROR); } - return result; + processService.forceProcessInstanceSuccessByTaskInstanceId(task); + log.info("Force success task instance:{} success", taskInstanceId); } @Override diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java index d2da5bc638..eae064bb24 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import java.text.MessageFormat; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; @@ -30,6 +32,12 @@ public class AssertionsHelper extends Assertions { Assertions.assertEquals(status.getCode(), exception.getCode()); } + public static void assertThrowsServiceException(String message, Executable executable) { + ServiceException exception = Assertions.assertThrows(ServiceException.class, executable); + Assertions.assertEquals(MessageFormat.format(Status.INTERNAL_SERVER_ERROR_ARGS.getMsg(), message), + exception.getMessage()); + } + public static void assertDoesNotThrow(Executable executable) { Assertions.assertDoesNotThrow(executable); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 7ebe5bf757..b58944537b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -82,9 +82,7 @@ public class TaskInstanceControllerTest extends AbstractControllerTest { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("taskInstanceId", "104"); - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); - when(taskInstanceService.forceTaskSuccess(any(User.class), anyLong(), anyInt())).thenReturn(mockResult); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(User.class), anyLong(), anyInt()); MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/task-instance/force-success", "cxc_1113") .header(SESSION_ID, sessionId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java index cc70f200e1..8e76ec1694 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.controller.AbstractControllerTest; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceQueryRequest; +import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceSuccessResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -85,12 +86,9 @@ public class TaskInstanceV2ControllerTest extends AbstractControllerTest { @Test public void testForceTaskSuccess() { - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt()); - when(taskInstanceService.forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt())).thenReturn(mockResult); - - Result taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); + TaskInstanceSuccessResponse taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); Assertions.assertEquals(Integer.valueOf(Status.SUCCESS.getCode()), taskResult.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java index aca0d80a6f..dd7acb16d5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.FORCED_SUCCESS; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_INSTANCE; import static org.mockito.ArgumentMatchers.any; @@ -25,7 +26,6 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; @@ -35,15 +35,16 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -65,7 +66,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.springframework.boot.test.context.SpringBootTest; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -74,7 +74,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -@SpringBootTest(classes = ApiApplicationServer.class) public class TaskInstanceServiceTest { @InjectMocks @@ -100,6 +99,8 @@ public class TaskInstanceServiceTest { @Mock TaskInstanceDao taskInstanceDao; + @Mock + ProcessInstanceDao workflowInstanceDao; @Test public void queryTaskListPaging() { @@ -324,6 +325,7 @@ public class TaskInstanceServiceTest { private TaskInstance getTaskInstance() { TaskInstance taskInstance = new TaskInstance(); taskInstance.setId(1); + taskInstance.setProjectCode(1L); taskInstance.setName("test_task_instance"); taskInstance.setStartTime(new Date()); taskInstance.setEndTime(new Date()); @@ -343,64 +345,69 @@ public class TaskInstanceServiceTest { } @Test - public void testForceTaskSuccess() { + public void testForceTaskSuccess_withNoPermission() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + .checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + assertThrowsServiceException(Status.USER_NO_OPERATION_PROJECT_PERM, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withTaskInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.empty()); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())).thenReturn(Optional.empty()); + + assertThrowsServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFinished() { User user = getAdminUser(); long projectCode = 1L; - Project project = getProject(projectCode); - int taskId = 1; TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, projectCode, FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); - Map mockSuccess = new HashMap<>(5); - putMsg(mockSuccess, Status.SUCCESS); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); + assertThrowsServiceException( + "The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance", + () -> taskInstanceService.forceTaskSuccess(user, projectCode, task.getId())); + } - // user auth failed - Map mockFailure = new HashMap<>(5); - putMsg(mockFailure, Status.USER_NO_OPERATION_PROJECT_PERM, user.getUserName(), projectCode); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockFailure); - Result authFailRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertNotSame(Status.SUCCESS.getCode(), authFailRes.getCode()); - - // test task not found - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockSuccess); - when(taskInstanceMapper.selectById(Mockito.anyInt())).thenReturn(null); - TaskDefinition taskDefinition = new TaskDefinition(); - taskDefinition.setProjectCode(projectCode); - when(taskDefinitionMapper.queryByCode(task.getTaskCode())).thenReturn(taskDefinition); - Result taskNotFoundRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), taskNotFoundRes.getCode().intValue()); - - // test task instance state error - task.setState(TaskExecutionStatus.SUCCESS); - when(taskInstanceMapper.selectById(1)).thenReturn(task); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result taskStateErrorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_STATE_OPERATION_ERROR.getCode(), - taskStateErrorRes.getCode().intValue()); - - // test error - task.setState(TaskExecutionStatus.FAILURE); - when(taskInstanceMapper.updateById(task)).thenReturn(0); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result errorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.FORCE_TASK_SUCCESS_ERROR.getCode(), errorRes.getCode().intValue()); - - // test success - task.setState(TaskExecutionStatus.FAILURE); - task.setEndTime(null); - when(taskInstanceMapper.updateById(task)).thenReturn(1); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result successRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.SUCCESS.getCode(), successRes.getCode().intValue()); - Assertions.assertNotNull(task.getEndTime()); + @Test + public void testForceTaskSuccess_withTaskInstanceNotFinished() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.FAILURE); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); + assertThrowsServiceException( + Status.TASK_INSTANCE_STATE_OPERATION_ERROR, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); } @Test diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 101350e90d..8787aabc8b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -189,7 +189,7 @@ public interface ProcessService { public String findConfigYamlByName(String clusterName); - void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId); + void forceProcessInstanceSuccessByTaskInstanceId(TaskInstance taskInstance); void saveCommandTrigger(Integer commandId, Integer processInstanceId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 028ab7651f..3c207ae982 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -276,6 +276,7 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private TriggerRelationService triggerRelationService; + /** * todo: split this method * handle Command (construct ProcessInstance from Command) , wrapped in transaction @@ -621,13 +622,13 @@ public class ProcessServiceImpl implements ProcessService { /** * Get workflow runtime tenant - * + *

* the workflow provides a tenant and uses the provided tenant; * when no tenant is provided or the provided tenant is the default tenant, \ * the user's tenant created by the workflow is used * * @param tenantCode tenantCode - * @param userId userId + * @param userId userId * @return tenant code */ @Override @@ -2114,11 +2115,7 @@ public class ProcessServiceImpl implements ProcessService { } @Override - public void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId) { - TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); - if (task == null) { - return; - } + public void forceProcessInstanceSuccessByTaskInstanceId(TaskInstance task) { ProcessInstance processInstance = findProcessInstanceDetailById(task.getProcessInstanceId()).orElse(null); if (processInstance != null && (processInstance.getState().isFailure() || processInstance.getState().isStop())) { @@ -2139,7 +2136,7 @@ public class ProcessServiceImpl implements ProcessService { List failTaskList = validTaskList.stream() .filter(instance -> instance.getState().isFailure() || instance.getState().isKill()) .map(TaskInstance::getId).collect(Collectors.toList()); - if (failTaskList.size() == 1 && failTaskList.contains(taskInstanceId)) { + if (failTaskList.size() == 1 && failTaskList.contains(task.getId())) { processInstance.setStateWithDesc(WorkflowExecutionStatus.SUCCESS, "success by task force success"); processInstanceDao.updateById(processInstance); } From ead54534a2061cc2daaf8ee24024a011a8b8ce4f Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 17:27:36 +0800 Subject: [PATCH 53/88] Fix QUARTZ table order is not correct in initialization schema (#15857) --- .../resources/sql/dolphinscheduler_mysql.sql | 130 +++++++++--------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index e2138b67a2..b30bc13887 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -15,7 +15,70 @@ * limitations under the License. */ -SET FOREIGN_KEY_CHECKS=0; +-- ---------------------------- +-- Table structure for QRTZ_JOB_DETAILS +-- ---------------------------- +DROP TABLE IF EXISTS `QRTZ_JOB_DETAILS`; +CREATE TABLE `QRTZ_JOB_DETAILS` ( + `SCHED_NAME` varchar(120) NOT NULL, + `JOB_NAME` varchar(200) NOT NULL, + `JOB_GROUP` varchar(200) NOT NULL, + `DESCRIPTION` varchar(250) DEFAULT NULL, + `JOB_CLASS_NAME` varchar(250) NOT NULL, + `IS_DURABLE` varchar(1) NOT NULL, + `IS_NONCONCURRENT` varchar(1) NOT NULL, + `IS_UPDATE_DATA` varchar(1) NOT NULL, + `REQUESTS_RECOVERY` varchar(1) NOT NULL, + `JOB_DATA` blob, + PRIMARY KEY (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_J_REQ_RECOVERY` (`SCHED_NAME`,`REQUESTS_RECOVERY`), + KEY `IDX_QRTZ_J_GRP` (`SCHED_NAME`,`JOB_GROUP`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; + +-- ---------------------------- +-- Records of QRTZ_JOB_DETAILS +-- ---------------------------- + +-- ---------------------------- +-- Table structure for QRTZ_TRIGGERS +-- ---------------------------- +DROP TABLE IF EXISTS `QRTZ_TRIGGERS`; +CREATE TABLE `QRTZ_TRIGGERS` ( + `SCHED_NAME` varchar(120) NOT NULL, + `TRIGGER_NAME` varchar(200) NOT NULL, + `TRIGGER_GROUP` varchar(200) NOT NULL, + `JOB_NAME` varchar(200) NOT NULL, + `JOB_GROUP` varchar(200) NOT NULL, + `DESCRIPTION` varchar(250) DEFAULT NULL, + `NEXT_FIRE_TIME` bigint(13) DEFAULT NULL, + `PREV_FIRE_TIME` bigint(13) DEFAULT NULL, + `PRIORITY` int(11) DEFAULT NULL, + `TRIGGER_STATE` varchar(16) NOT NULL, + `TRIGGER_TYPE` varchar(8) NOT NULL, + `START_TIME` bigint(13) NOT NULL, + `END_TIME` bigint(13) DEFAULT NULL, + `CALENDAR_NAME` varchar(200) DEFAULT NULL, + `MISFIRE_INSTR` smallint(2) DEFAULT NULL, + `JOB_DATA` blob, + PRIMARY KEY (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`), + KEY `IDX_QRTZ_T_J` (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_T_JG` (`SCHED_NAME`,`JOB_GROUP`), + KEY `IDX_QRTZ_T_C` (`SCHED_NAME`,`CALENDAR_NAME`), + KEY `IDX_QRTZ_T_G` (`SCHED_NAME`,`TRIGGER_GROUP`), + KEY `IDX_QRTZ_T_STATE` (`SCHED_NAME`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_N_STATE` (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_N_G_STATE` (`SCHED_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_NEXT_FIRE_TIME` (`SCHED_NAME`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_ST` (`SCHED_NAME`,`TRIGGER_STATE`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`), + KEY `IDX_QRTZ_T_NFT_ST_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_STATE`), + KEY `IDX_QRTZ_T_NFT_ST_MISFIRE_GRP` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), + CONSTRAINT `QRTZ_TRIGGERS_ibfk_1` FOREIGN KEY (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) REFERENCES `QRTZ_JOB_DETAILS` (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; + +-- ---------------------------- +-- Records of QRTZ_TRIGGERS +-- ---------------------------- -- ---------------------------- -- Table structure for QRTZ_BLOB_TRIGGERS @@ -99,30 +162,6 @@ CREATE TABLE `QRTZ_FIRED_TRIGGERS` ( -- Records of QRTZ_FIRED_TRIGGERS -- ---------------------------- --- ---------------------------- --- Table structure for QRTZ_JOB_DETAILS --- ---------------------------- -DROP TABLE IF EXISTS `QRTZ_JOB_DETAILS`; -CREATE TABLE `QRTZ_JOB_DETAILS` ( - `SCHED_NAME` varchar(120) NOT NULL, - `JOB_NAME` varchar(200) NOT NULL, - `JOB_GROUP` varchar(200) NOT NULL, - `DESCRIPTION` varchar(250) DEFAULT NULL, - `JOB_CLASS_NAME` varchar(250) NOT NULL, - `IS_DURABLE` varchar(1) NOT NULL, - `IS_NONCONCURRENT` varchar(1) NOT NULL, - `IS_UPDATE_DATA` varchar(1) NOT NULL, - `REQUESTS_RECOVERY` varchar(1) NOT NULL, - `JOB_DATA` blob, - PRIMARY KEY (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_J_REQ_RECOVERY` (`SCHED_NAME`,`REQUESTS_RECOVERY`), - KEY `IDX_QRTZ_J_GRP` (`SCHED_NAME`,`JOB_GROUP`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - --- ---------------------------- --- Records of QRTZ_JOB_DETAILS --- ---------------------------- - -- ---------------------------- -- Table structure for QRTZ_LOCKS -- ---------------------------- @@ -213,47 +252,6 @@ CREATE TABLE `QRTZ_SIMPROP_TRIGGERS` ( -- Records of QRTZ_SIMPROP_TRIGGERS -- ---------------------------- --- ---------------------------- --- Table structure for QRTZ_TRIGGERS --- ---------------------------- -DROP TABLE IF EXISTS `QRTZ_TRIGGERS`; -CREATE TABLE `QRTZ_TRIGGERS` ( - `SCHED_NAME` varchar(120) NOT NULL, - `TRIGGER_NAME` varchar(200) NOT NULL, - `TRIGGER_GROUP` varchar(200) NOT NULL, - `JOB_NAME` varchar(200) NOT NULL, - `JOB_GROUP` varchar(200) NOT NULL, - `DESCRIPTION` varchar(250) DEFAULT NULL, - `NEXT_FIRE_TIME` bigint(13) DEFAULT NULL, - `PREV_FIRE_TIME` bigint(13) DEFAULT NULL, - `PRIORITY` int(11) DEFAULT NULL, - `TRIGGER_STATE` varchar(16) NOT NULL, - `TRIGGER_TYPE` varchar(8) NOT NULL, - `START_TIME` bigint(13) NOT NULL, - `END_TIME` bigint(13) DEFAULT NULL, - `CALENDAR_NAME` varchar(200) DEFAULT NULL, - `MISFIRE_INSTR` smallint(2) DEFAULT NULL, - `JOB_DATA` blob, - PRIMARY KEY (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`), - KEY `IDX_QRTZ_T_J` (`SCHED_NAME`,`JOB_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_T_JG` (`SCHED_NAME`,`JOB_GROUP`), - KEY `IDX_QRTZ_T_C` (`SCHED_NAME`,`CALENDAR_NAME`), - KEY `IDX_QRTZ_T_G` (`SCHED_NAME`,`TRIGGER_GROUP`), - KEY `IDX_QRTZ_T_STATE` (`SCHED_NAME`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_N_STATE` (`SCHED_NAME`,`TRIGGER_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_N_G_STATE` (`SCHED_NAME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_NEXT_FIRE_TIME` (`SCHED_NAME`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_ST` (`SCHED_NAME`,`TRIGGER_STATE`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`), - KEY `IDX_QRTZ_T_NFT_ST_MISFIRE` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_STATE`), - KEY `IDX_QRTZ_T_NFT_ST_MISFIRE_GRP` (`SCHED_NAME`,`MISFIRE_INSTR`,`NEXT_FIRE_TIME`,`TRIGGER_GROUP`,`TRIGGER_STATE`), - CONSTRAINT `QRTZ_TRIGGERS_ibfk_1` FOREIGN KEY (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) REFERENCES `QRTZ_JOB_DETAILS` (`SCHED_NAME`, `JOB_NAME`, `JOB_GROUP`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - --- ---------------------------- --- Records of QRTZ_TRIGGERS --- ---------------------------- - -- ---------------------------- -- Table structure for t_ds_access_token -- ---------------------------- From 9437d276e76b18a2cf6c469fa886a027115ece03 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 16 Apr 2024 22:49:11 +0800 Subject: [PATCH 54/88] Change ssh heartbeat type to IGNORE (#15858) --- .../dolphinscheduler/plugin/datasource/ssh/SSHUtils.java | 2 +- .../plugin/task/remoteshell/RemoteExecutor.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java index 42e1175e2e..7abb18cee8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java @@ -59,7 +59,7 @@ public class SSHUtils { throw new Exception("Failed to add public key identity", e); } } - session.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.RESERVED, Duration.ofSeconds(3)); + session.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.IGNORE, Duration.ofSeconds(3)); return session; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java index 334ebb6195..307023043a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java @@ -141,7 +141,6 @@ public class RemoteExecutor implements AutoCloseable { } } cleanData(taskId); - log.error("Remote shell task failed"); return exitCode; } @@ -232,8 +231,10 @@ public class RemoteExecutor implements AutoCloseable { channel.open(); channel.waitFor(EnumSet.of(ClientChannelEvent.CLOSED), 0); channel.close(); - if (channel.getExitStatus() != 0) { - throw new TaskException("Remote shell task error, error message: " + err.toString()); + Integer exitStatus = channel.getExitStatus(); + if (exitStatus == null || exitStatus != 0) { + throw new TaskException( + "Remote shell task error, exitStatus: " + exitStatus + " error message: " + err); } return out.toString(); } From 9a48aca83c7275c934a08eab3c0bf4203a80149d Mon Sep 17 00:00:00 2001 From: zuo <58384836+xxzuo@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:18:29 +0800 Subject: [PATCH 55/88] [Fix-15866][Doc] update the taobao mirror link (#15867) --- docs/docs/en/contribute/frontend-development.md | 2 +- docs/docs/en/faq.md | 4 ++-- docs/docs/zh/contribute/frontend-development.md | 2 +- docs/docs/zh/faq.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/en/contribute/frontend-development.md b/docs/docs/en/contribute/frontend-development.md index 0cc86811f8..50f8c91fb4 100644 --- a/docs/docs/en/contribute/frontend-development.md +++ b/docs/docs/en/contribute/frontend-development.md @@ -33,7 +33,7 @@ Use the command line mode `cd` enter the `dolphinscheduler-ui` project director > If `npm install` is very slow, you can set the taobao mirror ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend: diff --git a/docs/docs/en/faq.md b/docs/docs/en/faq.md index b0954b3468..7ac4afb76e 100644 --- a/docs/docs/en/faq.md +++ b/docs/docs/en/faq.md @@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory sudo rm -rf node_modules ``` -​ 2, install node-sass through npm.taobao.org +​ 2, install node-sass through npmmirror.com ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md) diff --git a/docs/docs/zh/contribute/frontend-development.md b/docs/docs/zh/contribute/frontend-development.md index 42eb2973dd..249b17d546 100644 --- a/docs/docs/zh/contribute/frontend-development.md +++ b/docs/docs/zh/contribute/frontend-development.md @@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2 > 如果 `npm install` 速度非常慢，你可以设置淘宝镜像 ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互： diff --git a/docs/docs/zh/faq.md b/docs/docs/zh/faq.md index 13e0dc8b84..a4f32d08a8 100644 --- a/docs/docs/zh/faq.md +++ b/docs/docs/zh/faq.md @@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录 sudo rm -rf node_modules ``` -​ 2，通过 npm.taobao.org 下载 node-sass +​ 2，通过 npmmirror.com 下载 node-sass ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md) From 3fe9fd45b12e2e6d190729e64b2c4663b0752ada Mon Sep 17 00:00:00 2001 From: XinXing <49302071+xinxingi@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:36:18 +0800 Subject: [PATCH 56/88] [Fix-15706] Seatunnel improvement (#15852) * fix_seatunnel_15706 * CodeFormat * Change to use JSONUtils * Constants moved to constant code * Delete empty lines * Delete empty lines --- .../common/utils/JSONUtils.java | 6 +- .../plugin/task/seatunnel/Constants.java | 2 + .../plugin/task/seatunnel/SeatunnelTask.java | 9 +- .../task/seatunnel/SeatunnelTaskTest.java | 93 +++++++++++++++++++ 4 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 6750b364f9..bfc3af2c58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -196,7 +196,10 @@ public final class JSONUtils { * @return true if valid */ public static boolean checkJsonValid(String json) { + return checkJsonValid(json, true); + } + public static boolean checkJsonValid(String json, Boolean logFlag) { if (Strings.isNullOrEmpty(json)) { return false; } @@ -205,7 +208,8 @@ public final class JSONUtils { objectMapper.readTree(json); return true; } catch (IOException e) { - log.error("check json object valid exception!", e); + if (logFlag) + log.error("check json object valid exception!", e); } return false; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java index fb1c52ee18..5f0f22b0a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/Constants.java @@ -29,5 +29,7 @@ public class Constants { public static final String STARTUP_SCRIPT_SPARK = "spark"; public static final String STARTUP_SCRIPT_FLINK = "flink"; public static final String STARTUP_SCRIPT_SEATUNNEL = "seatunnel"; + public static final String JSON_SUFFIX = "json"; + public static final String CONF_SUFFIX = "conf"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java index 547e0158ef..2aadab8039 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java @@ -184,8 +184,13 @@ public class SeatunnelTask extends AbstractRemoteTask { } private String buildConfigFilePath() { - return String.format("%s/seatunnel_%s.conf", taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId()); + return String.format("%s/seatunnel_%s.%s", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId(), formatDetector()); + } + + private String formatDetector() { + return JSONUtils.checkJsonValid(seatunnelParameters.getRawScript(), false) ? Constants.JSON_SUFFIX + : Constants.CONF_SUFFIX; } private void createConfigFileIfNotExists(String script, String scriptFile) throws IOException { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java new file mode 100644 index 0000000000..8f4c2a815a --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/test/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.task.seatunnel; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +public class SeatunnelTaskTest { + private static final String EXECUTE_PATH = "/home"; + private static final String TASK_APPID = "9527"; + + @Test + public void formatDetector() throws Exception{ + SeatunnelParameters seatunnelParameters = new SeatunnelParameters(); + seatunnelParameters.setRawScript(RAW_SCRIPT); + + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setExecutePath(EXECUTE_PATH); + taskExecutionContext.setTaskAppId(TASK_APPID); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(seatunnelParameters)); + + SeatunnelTask seatunnelTask = new SeatunnelTask(taskExecutionContext); + seatunnelTask.setSeatunnelParameters(seatunnelParameters); + Assertions.assertEquals("/home/seatunnel_9527.conf", seatunnelTask.buildCustomConfigCommand()); + + seatunnelParameters.setRawScript(RAW_SCRIPT_2); + seatunnelTask.setSeatunnelParameters(seatunnelParameters); + Assertions.assertEquals("/home/seatunnel_9527.json", seatunnelTask.buildCustomConfigCommand()); + } + private static final String RAW_SCRIPT = "env {\n" + + " execution.parallelism = 2\n" + + " job.mode = \"BATCH\"\n" + + " checkpoint.interval = 10000\n" + + "}\n" + + "\n" + + "source {\n" + + " FakeSource {\n" + + " parallelism = 2\n" + + " result_table_name = \"fake\"\n" + + " row.num = 16\n" + + " schema = {\n" + + " fields {\n" + + " name = \"string\"\n" + + " age = \"int\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "\n" + + "sink {\n" + + " Console {\n" + + " }\n" + + "}"; + private static final String RAW_SCRIPT_2 = "{\n" + + " \"env\": {\n" + + " \"execution.parallelism\": 2,\n" + + " \"job.mode\": \"BATCH\",\n" + + " \"checkpoint.interval\": 10000\n" + + " },\n" + + " \"source\": {\n" + + " \"FakeSource\": {\n" + + " \"parallelism\": 2,\n" + + " \"result_table_name\": \"fake\",\n" + + " \"row.num\": 16,\n" + + " \"schema\": {\n" + + " \"fields\": {\n" + + " \"name\": \"string\",\n" + + " \"age\": \"int\"\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sink\": {\n" + + " \"Console\": {}\n" + + " }\n" + + "}"; +} \ No newline at end of file From a8bc23748d7e4371e962c095d40e06ce88fdf497 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 11:31:11 +0800 Subject: [PATCH 57/88] Fix write audit log error will cause the request failed (#15868) --- .../api/audit/OperatorLogAspect.java | 16 ++++++++++-- .../api/audit/OperatorUtils.java | 1 - .../api/audit/operator/AuditOperator.java | 13 +++++++--- .../api/audit/operator/BaseAuditOperator.java | 26 +++++++++---------- .../api/service/AuditService.java | 10 ------- .../api/service/impl/AuditServiceImpl.java | 8 ------ 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java index 4e15112980..da8a7bd3e6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.api.audit; import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.lang.reflect.Method; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -54,8 +56,18 @@ public class OperatorLogAspect { log.warn("Operation is null of method: {}", method.getName()); return point.proceed(); } + long beginTime = System.currentTimeMillis(); - AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); - return operator.recordAudit(point, operation.description(), operatorLog.auditType()); + Map paramsMap = OperatorUtils.getParamsMap(point, signature); + Result result = (Result) point.proceed(); + try { + AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); + long latency = System.currentTimeMillis() - beginTime; + operator.recordAudit(paramsMap, result, latency, operation, operatorLog); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } + + return result; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index a0233c7e93..10ccf8d648 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -66,7 +66,6 @@ public class OperatorUtils { auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); auditLog.setCreateTime(new Date()); - auditLogList.add(auditLog); return auditLogList; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java index 6e1c016313..7d76d5c4bd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java @@ -17,11 +17,18 @@ package org.apache.dolphinscheduler.api.audit.operator; -import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.utils.Result; -import org.aspectj.lang.ProceedingJoinPoint; +import java.util.Map; + +import io.swagger.v3.oas.annotations.Operation; public interface AuditOperator { - Object recordAudit(ProceedingJoinPoint point, String describe, AuditType auditType) throws Throwable; + void recordAudit(Map paramsMap, + Result result, + long latency, + Operation operation, + OperatorLog operatorLog) throws Throwable; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java index 5c896a058a..270a5e4df4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.audit.operator; +import org.apache.dolphinscheduler.api.audit.OperatorLog; import org.apache.dolphinscheduler.api.audit.OperatorUtils; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.service.AuditService; @@ -31,12 +32,11 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.reflect.MethodSignature; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.base.Strings; +import io.swagger.v3.oas.annotations.Operation; @Service @Slf4j @@ -46,26 +46,27 @@ public abstract class BaseAuditOperator implements AuditOperator { private AuditService auditService; @Override - public Object recordAudit(ProceedingJoinPoint point, String describe, AuditType auditType) throws Throwable { - long beginTime = System.currentTimeMillis(); + public void recordAudit(Map paramsMap, + Result result, + long latency, + Operation operation, + OperatorLog operatorLog) { - MethodSignature signature = (MethodSignature) point.getSignature(); - Map paramsMap = OperatorUtils.getParamsMap(point, signature); + AuditType auditType = operatorLog.auditType(); User user = OperatorUtils.getUser(paramsMap); if (user == null) { log.error("user is null"); - return point.proceed(); + return; } - List auditLogList = OperatorUtils.buildAuditLogList(describe, auditType, user); + List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); setRequestParam(auditType, auditLogList, paramsMap); - Result result = (Result) point.proceed(); if (OperatorUtils.resultFail(result)) { log.error("request fail, code {}", result.getCode()); - return result; + return; } setObjectIdentityFromReturnObject(auditType, result, auditLogList); @@ -73,10 +74,9 @@ public abstract class BaseAuditOperator implements AuditOperator { modifyAuditOperationType(auditType, paramsMap, auditLogList); modifyAuditObjectType(auditType, paramsMap, auditLogList); - long latency = System.currentTimeMillis() - beginTime; - auditService.addAudit(auditLogList, latency); + auditLogList.forEach(auditLog -> auditLog.setLatency(latency)); + auditLogList.forEach(auditLog -> auditService.addAudit(auditLog)); - return result; } protected void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java index b4296452f6..8fb99eaf73 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.api.dto.AuditDto; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import java.util.List; - /** * audit information service */ @@ -35,14 +33,6 @@ public interface AuditService { */ void addAudit(AuditLog auditLog); - /** - * add audit by list - * - * @param auditLogList auditLog list - * @param latency api latency milliseconds - */ - void addAudit(List auditLogList, long latency); - /** * query audit log list * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index 2ec547bcb6..f8f763b4f4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -57,14 +57,6 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { auditLogMapper.insert(auditLog); } - @Override - public void addAudit(List auditLogList, long latency) { - auditLogList.forEach(auditLog -> { - auditLog.setLatency(latency); - addAudit(auditLog); - }); - } - /** * query audit log paging * From 163f8f01f0b3726374c81d9fa87da6571b3c460c Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 15:43:28 +0800 Subject: [PATCH 58/88] Fix jdbc registry cannot work (#15861) --- .github/workflows/backend.yml | 12 +- .../Dockerfile | 6 +- .../mysql_with_mysql_registry/deploy.sh | 44 +++++ .../docker-compose-base.yaml | 34 ++++ .../docker-compose-cluster.yaml | 0 .../dolphinscheduler_env.sh | 58 +++++++ .../mysql_with_mysql_registry/install_env.sh | 58 +++++++ .../running_test.sh | 0 .../start-job.sh | 8 +- .../mysql_with_zookeeper_registry/Dockerfile | 48 ++++++ .../deploy.sh | 0 .../docker-compose-base.yaml | 0 .../docker-compose-cluster.yaml | 29 ++++ .../dolphinscheduler_env.sh | 0 .../install_env.sh | 0 .../running_test.sh | 108 +++++++++++++ .../start-job.sh | 33 ++++ .../Dockerfile | 39 +++++ .../deploy.sh | 41 +++++ .../docker-compose-base.yaml | 35 ++++ .../docker-compose-cluster.yaml | 0 .../dolphinscheduler_env.sh | 58 +++++++ .../install_env.sh | 58 +++++++ .../running_test.sh | 0 .../start-job.sh | 33 ++++ .../Dockerfile | 6 +- .../deploy.sh | 0 .../docker-compose-base.yaml | 0 .../docker-compose-cluster.yaml | 29 ++++ .../dolphinscheduler_env.sh | 0 .../install_env.sh | 0 .../running_test.sh | 109 +++++++++++++ .../start-job.sh | 8 +- .../dolphinscheduler/alert/AlertServer.java | 19 ++- .../api/ApiApplicationServer.java | 6 +- .../dolphinscheduler/dao/PluginDao.java | 4 +- .../{ => repository/impl}/AlertDaoTest.java | 4 +- .../server/master/MasterServer.java | 6 +- .../dolphinscheduler-registry-jdbc/README.md | 10 +- .../dolphinscheduler-registry-jdbc/pom.xml | 10 ++ ...ava => JdbcRegistryAutoConfiguration.java} | 30 +++- .../main/resources/META-INF/spring.factories | 19 +++ .../main/resources/mysql_registry_init.sql | 1 - .../src/main/bin/initialize-jdbc-registry.sh | 31 ++++ .../tools/command/CommandApplication.java | 152 ++++++++++++++++++ .../src/main/resources/application.yaml | 2 + .../server/worker/WorkerServer.java | 6 +- 47 files changed, 1111 insertions(+), 43 deletions(-) rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/Dockerfile (85%) create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/docker-compose-cluster.yaml (100%) create mode 100755 .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh create mode 100644 .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/running_test.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_mysql_registry}/start-job.sh (74%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/deploy.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/docker-compose-base.yaml (100%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/dolphinscheduler_env.sh (100%) rename .github/workflows/cluster-test/{mysql => mysql_with_zookeeper_registry}/install_env.sh (100%) create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh create mode 100644 .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml rename .github/workflows/cluster-test/{postgresql => postgresql_with_postgresql_registry}/docker-compose-cluster.yaml (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_postgresql_registry}/running_test.sh (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/Dockerfile (77%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/deploy.sh (100%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/docker-compose-base.yaml (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/dolphinscheduler_env.sh (100%) rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/install_env.sh (100%) create mode 100644 .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh rename .github/workflows/cluster-test/{postgresql => postgresql_with_zookeeper_registry}/start-job.sh (73%) rename dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/{ => repository/impl}/AlertDaoTest.java (95%) rename dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/{JdbcRegistryConfiguration.java => JdbcRegistryAutoConfiguration.java} (65%) create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh create mode 100644 dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ea09a17fd2..671223d286 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -106,10 +106,14 @@ jobs: strategy: matrix: case: - - name: cluster-test-mysql - script: .github/workflows/cluster-test/mysql/start-job.sh - - name: cluster-test-postgresql - script: .github/workflows/cluster-test/postgresql/start-job.sh + - name: cluster-test-mysql-with-zookeeper-registry + script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh + - name: cluster-test-mysql-with-mysql-registry + script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh + - name: cluster-test-postgresql-zookeeper-registry + script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh + - name: cluster-test-postgresql-with-postgresql-registry + script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/cluster-test/mysql/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile similarity index 85% rename from .github/workflows/cluster-test/mysql/Dockerfile rename to .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile index c7d6abe889..12c7db3c18 100644 --- a/.github/workflows/cluster-test/mysql/Dockerfile +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile @@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Download mysql jar ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" @@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER #Deploy -COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh new file mode 100644 index 0000000000..72b2a630fa --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Create database +mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..d59e3c868c --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + mysql: + container_name: mysql + image: mysql:5.7.36 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: 123456 + ports: + - "3306:3306" + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + timeout: 60s + retries: 120 diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh new file mode 100755 index 0000000000..58937e740c --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export SPRING_DATASOURCE_USERNAME=root +export SPRING_DATASOURCE_PASSWORD=123456 + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export REGISTRY_HIKARI_CONFIG_USERNAME=root +export REGISTRY_HIKARI_CONFIG_PASSWORD=123456 + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql/running_test.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/running_test.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/mysql/start-job.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh similarity index 74% rename from .github/workflows/cluster-test/mysql/start-job.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh index ee67c5179b..0ce48c64ae 100644 --- a/.github/workflows/cluster-test/mysql/start-job.sh +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d #Build ds mysql cluster image -docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile . +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile . #Start ds mysql cluster container -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/mysql/running_test.sh +/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile new file mode 100644 index 0000000000..574c059442 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Download mysql jar +ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER + +#Deploy +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql/deploy.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/deploy.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/mysql/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-base.yaml rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..7343c8eee7 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_mysql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/mysql/install_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/install_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..7582c3ccc5 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh new file mode 100644 index 0000000000..db8d23147e --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d + +#Build ds mysql cluster image +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile . + +#Start ds mysql cluster container +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile new file mode 100644 index 0000000000..bb2d9a5383 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Deploy +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh new file mode 100644 index 0000000000..37bf3433c0 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..1793d94f39 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + postgres: + container_name: postgres + image: postgres:14.1 + restart: always + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dolphinscheduler + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh new file mode 100644 index 0000000000..e7fd1b7204 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-postgresql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export SPRING_DATASOURCE_USERNAME=postgres +export SPRING_DATASOURCE_PASSWORD=postgres + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=jdbc +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export REGISTRY_HIKARI_CONFIG_USERNAME=postgres +export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/running_test.sh rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh new file mode 100644 index 0000000000..e2b6b630e8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d + +#Build ds postgresql cluster image +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile . + +#Start ds postgresql cluster container +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile similarity index 77% rename from .github/workflows/cluster-test/postgresql/Dockerfile rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile index 38234ee7b3..077b5c97b8 100644 --- a/.github/workflows/cluster-test/postgresql/Dockerfile +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile @@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Deploy -COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql/deploy.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/deploy.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-base.yaml rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..9ab79ea44d --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_postgresql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/install_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..0bc861c389 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh similarity index 73% rename from .github/workflows/cluster-test/postgresql/start-job.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh index ba0878e3ec..fe755c97f1 100644 --- a/.github/workflows/cluster-test/postgresql/start-job.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d #Build ds postgresql cluster image -docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile . +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile . #Start ds postgresql cluster container -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh +/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index fd3d4b02e3..ff0088ea8d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -27,21 +27,24 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.event.EventListener; +import org.springframework.context.annotation.FilterType; -@SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") @Slf4j +@SpringBootApplication +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) public class AlertServer { @Autowired @@ -59,11 +62,11 @@ public class AlertServer { AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); Thread.currentThread().setName(Constants.THREAD_NAME_ALERT_SERVER); - new SpringApplicationBuilder(AlertServer.class).run(args); + SpringApplication.run(AlertServer.class, args); } - @EventListener - public void run(ApplicationReadyEvent readyEvent) { + @PostConstruct + public void run() { log.info("Alert server is staring ..."); alertPluginManager.start(); alertRegistryClient.start(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index c7e6d9778f..20a6412076 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; @@ -38,11 +39,14 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.servlet.ServletComponentScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.context.event.EventListener; @ServletComponentScan @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @Slf4j public class ApiApplicationServer { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java index 71e3be70c4..24cb022881 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java @@ -29,10 +29,10 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Repository; @Slf4j -@Component +@Repository public class PluginDao { @Autowired diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java similarity index 95% rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java rename to dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index f2cb503d9b..c0a841c1a0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao; +package org.apache.dolphinscheduler.dao.repository.impl; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.ProfileType; +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.entity.Alert; import java.util.List; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 752479e600..fd262a8c6e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.scheduler.api.SchedulerApi; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; @@ -46,10 +47,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @EnableTransactionManagement @EnableCaching @Slf4j diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md index 3b1a2cb24f..554c375218 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md @@ -1,6 +1,7 @@ # Introduction -This module is the jdbc registry plugin module, this plugin will use jdbc as the registry center. Will use the database configuration same as DolphinScheduler in api'yaml default. +This module is the jdbc registry plugin module, this plugin will use jdbc as the registry center. Will use the database +configuration same as DolphinScheduler in api'yaml default. # How to use @@ -22,8 +23,11 @@ registry: After do this two steps, you can start your DolphinScheduler cluster, your cluster will use mysql as registry center to store server metadata. -NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use mysql database, since this plugin will not bundle this driver in distribution. -You can get the detail about Initialize the Database. +NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use mysql database, since this plugin will not +bundle this driver in distribution. +You can get the detail +about Initialize the +Database. ## Optional configuration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml index 47b6449293..d4285edfbd 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml @@ -62,6 +62,16 @@ mybatis-plus + + com.baomidou + mybatis-plus-boot-starter + + + org.apache.logging.log4j + log4j-to-slf4j + + + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java similarity index 65% rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java index 7b37749ab7..09211f99fb 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java @@ -22,40 +22,56 @@ import org.apache.dolphinscheduler.plugin.registry.jdbc.mapper.JdbcRegistryLockM import org.apache.ibatis.session.SqlSessionFactory; +import lombok.extern.slf4j.Slf4j; + import org.mybatis.spring.SqlSessionTemplate; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration; import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import com.zaxxer.hikari.HikariDataSource; -@Configuration +@Slf4j +@Configuration(proxyBeanMethods = false) +@MapperScan("org.apache.dolphinscheduler.plugin.registry.jdbc.mapper") @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") -public class JdbcRegistryConfiguration { +@AutoConfigureAfter(MybatisPlusAutoConfiguration.class) +public class JdbcRegistryAutoConfiguration { + + public JdbcRegistryAutoConfiguration() { + log.info("Load JdbcRegistryAutoConfiguration"); + } @Bean - @ConditionalOnProperty(prefix = "registry.hikari-config", name = "jdbc-url") - public SqlSessionFactory jdbcRegistrySqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties) throws Exception { + @ConditionalOnMissingBean + public SqlSessionFactory sqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties) throws Exception { + log.info("Initialize jdbcRegistrySqlSessionFactory"); MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); sqlSessionFactoryBean.setDataSource(new HikariDataSource(jdbcRegistryProperties.getHikariConfig())); return sqlSessionFactoryBean.getObject(); } @Bean - public SqlSessionTemplate jdbcRegistrySqlSessionTemplate(SqlSessionFactory jdbcRegistrySqlSessionFactory) { - jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryDataMapper.class); - jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryLockMapper.class); + @ConditionalOnMissingBean + public SqlSessionTemplate sqlSessionTemplate(SqlSessionFactory jdbcRegistrySqlSessionFactory) { + log.info("Initialize jdbcRegistrySqlSessionTemplate"); return new SqlSessionTemplate(jdbcRegistrySqlSessionFactory); } @Bean public JdbcRegistryDataMapper jdbcRegistryDataMapper(SqlSessionTemplate jdbcRegistrySqlSessionTemplate) { + jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryDataMapper.class); return jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryDataMapper.class); } @Bean public JdbcRegistryLockMapper jdbcRegistryLockMapper(SqlSessionTemplate jdbcRegistrySqlSessionTemplate) { + jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryLockMapper.class); return jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryLockMapper.class); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..aabe7e3252 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql index 30af3066ff..6df206b391 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql @@ -15,7 +15,6 @@ * limitations under the License. */ -SET FOREIGN_KEY_CHECKS = 0; DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`; CREATE TABLE `t_ds_jdbc_registry_data` diff --git a/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh new file mode 100644 index 0000000000..895d62a0bb --- /dev/null +++ b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=${DATABASE} \ + org.apache.dolphinscheduler.tools.command.CommandApplication diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java new file mode 100644 index 0000000000..1e419f4d19 --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.command; + +import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Component; + +import com.baomidou.mybatisplus.annotation.DbType; + +// todo: use spring-shell to manage the command +@SpringBootApplication +@ImportAutoConfiguration(DaoConfiguration.class) +public class CommandApplication { + + public static void main(String[] args) { + SpringApplication.run(CommandApplication.class, args); + } + + @Component + @Slf4j + static class JdbcRegistrySchemaInitializeCommand implements CommandLineRunner { + + @Autowired + private DatabaseDialect databaseDialect; + + @Autowired + private DbType dbType; + + @Autowired + private DataSource dataSource; + + JdbcRegistrySchemaInitializeCommand() { + } + + @Override + public void run(String... args) throws Exception { + if (databaseDialect.tableExists("t_ds_jdbc_registry_data") + || databaseDialect.tableExists("t_ds_jdbc_registry_lock")) { + log.warn("t_ds_jdbc_registry_data/t_ds_jdbc_registry_lock already exists"); + return; + } + if (dbType == DbType.MYSQL) { + jdbcRegistrySchemaInitializeInMysql(); + } else if (dbType == DbType.POSTGRE_SQL) { + jdbcRegistrySchemaInitializeInPG(); + } else { + log.error("Unsupported database type: {}", dbType); + } + } + + private void jdbcRegistrySchemaInitializeInMysql() throws SQLException { + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE `t_ds_jdbc_registry_data`\n" + + "(\n" + + " `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',\n" + + " `data_key` varchar(256) NOT NULL COMMENT 'key, like zookeeper node path',\n" + + " `data_value` text NOT NULL COMMENT 'data, like zookeeper node value',\n" + + " `data_type` tinyint(4) NOT NULL COMMENT '1: ephemeral node, 2: persistent node',\n" + + + " `last_term` bigint NOT NULL COMMENT 'last term time',\n" + + " `last_update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n" + + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',\n" + + + " PRIMARY KEY (`id`),\n" + + " unique (`data_key`)\n" + + ") ENGINE = InnoDB\n" + + " DEFAULT CHARSET = utf8;"); + + statement.execute("CREATE TABLE `t_ds_jdbc_registry_lock`\n" + + "(\n" + + " `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',\n" + + " `lock_key` varchar(256) NOT NULL COMMENT 'lock path',\n" + + " `lock_owner` varchar(256) NOT NULL COMMENT 'the lock owner, ip_processId',\n" + + " `last_term` bigint NOT NULL COMMENT 'last term time',\n" + + " `last_update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n" + + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',\n" + + + " PRIMARY KEY (`id`),\n" + + " unique (`lock_key`)\n" + + ") ENGINE = InnoDB\n" + + " DEFAULT CHARSET = utf8;"); + } + } + + private void jdbcRegistrySchemaInitializeInPG() throws SQLException { + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("create table t_ds_jdbc_registry_data\n" + + "(\n" + + " id serial\n" + + " constraint t_ds_jdbc_registry_data_pk primary key,\n" + + " data_key varchar not null,\n" + + " data_value text not null,\n" + + " data_type int4 not null,\n" + + " last_term bigint not null,\n" + + " last_update_time timestamp default current_timestamp not null,\n" + + " create_time timestamp default current_timestamp not null\n" + + ");"); + statement.execute( + "create unique index t_ds_jdbc_registry_data_key_uindex on t_ds_jdbc_registry_data (data_key);"); + statement.execute("create table t_ds_jdbc_registry_lock\n" + + "(\n" + + " id serial\n" + + " constraint t_ds_jdbc_registry_lock_pk primary key,\n" + + " lock_key varchar not null,\n" + + " lock_owner varchar not null,\n" + + " last_term bigint not null,\n" + + " last_update_time timestamp default current_timestamp not null,\n" + + " create_time timestamp default current_timestamp not null\n" + + ");"); + statement.execute( + "create unique index t_ds_jdbc_registry_lock_key_uindex on t_ds_jdbc_registry_lock (lock_key);"); + } + } + + } +} diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml index 38752021dc..136a4c5fd4 100644 --- a/dolphinscheduler-tools/src/main/resources/application.yaml +++ b/dolphinscheduler-tools/src/main/resources/application.yaml @@ -63,6 +63,8 @@ spring: datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + username: root + password: root --- spring: diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 86e755fc8a..b0866f7fd8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; @@ -47,11 +48,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; @SpringBootApplication @EnableTransactionManagement -@ComponentScan(basePackages = "org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) +}) @Slf4j public class WorkerServer implements IStoppable { From 8fc204940f7be5cf356f63dd223bc599f857fe69 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 18 Apr 2024 16:50:50 +0800 Subject: [PATCH 59/88] Add DSIP template (#15871) --- .github/ISSUE_TEMPLATE/dsip-request.yml | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/dsip-request.yml diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml new file mode 100644 index 0000000000..0908bba9e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dsip-request.yml @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Feature request +description: Suggest an idea for this project +title: "[DSIP-][Module Name] DSIP title" +labels: [ "DSIP", "Waiting for reply" ] +body: + - type: markdown + attributes: + value: | + For better global communication, Please write in English. + + If you feel the description in English is not clear, then you can append description in Chinese, thanks! + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first + to see whether the same DSIP was created already. + options: + - label: > + I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no + similar DSIP. + required: true + + - type: textarea + attributes: + label: Motivation + description: Why you want to do this change? + + - type: textarea + attributes: + label: Design Detail + description: Your design. + placeholder: > + It's better to provide a detailed design, such as the design of the interface, the design of the database, etc. + + - type: textarea + attributes: + label: Compatibility, Deprecation, and Migration Plan + description: > + If this feature is related to compatibility, deprecation, or migration, please describe it here. + + - type: textarea + attributes: + label: Test Plan + description: > + How to test this improvement. + + - type: checkboxes + attributes: + label: Code of Conduct + description: | + The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. + options: + - label: | + I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) + required: true + + - type: markdown + attributes: + value: "Thanks for completing our form!" From 325bfa821f9cd82ee15b681e8a5ae30d4976f8f1 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Thu, 18 Apr 2024 20:40:05 +0800 Subject: [PATCH 60/88] [TEST] increase cov of logger service (#15870) Co-authored-by: abzymeinsjtu Co-authored-by: Eric Gao --- .../api/service/impl/LoggerServiceImpl.java | 3 +- .../api/service/LoggerServiceTest.java | 62 ++++++++++++++++++- .../api/utils/ServiceTestUtil.java | 18 ++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index 0663b88374..c0ecb0e9b5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -237,7 +237,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService host, Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - byte[] logBytes = new byte[0]; + byte[] logBytes; ILogService iLogService = SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(), ILogService.class); @@ -251,6 +251,5 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService log.error("Download TaskInstance: {} Log Error", taskInstance.getName(), ex); throw new ServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR); } - } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 2c4de2ab7e..4861e1004e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -18,8 +18,10 @@ package org.apache.dolphinscheduler.api.service; import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; @@ -109,11 +111,25 @@ public class LoggerServiceTest { @Override public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { - return new TaskInstanceLogFileDownloadResponse(new byte[0]); + if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 1) { + return new TaskInstanceLogFileDownloadResponse(new byte[0]); + } else if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogFileDownloadResponse("log content".getBytes()); + } + + throw new ServiceException("download error"); } @Override public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() != null) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogPageQueryResponse("log content"); + } + } + return new TaskInstanceLogPageQueryResponse(); } @@ -177,6 +193,13 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); result = loggerService.queryLog(loginUser, 1, 1, 1); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + result = loggerService.queryLog(loginUser, 1, 0, 1); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + taskInstance.setLogPath(""); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, 1, 1, 1)); } @Test @@ -237,9 +260,15 @@ public class LoggerServiceTest { loginUser.setUserType(UserType.GENERAL_USER); TaskInstance taskInstance = new TaskInstance(); when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); + when(taskInstanceDao.queryById(10)).thenReturn(null); + + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); + TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(projectCode); taskDefinition.setCode(1L); + // SUCCESS taskInstance.setTaskCode(1L); taskInstance.setId(1); @@ -249,13 +278,27 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(10); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(1); + taskInstance.setId(10); + when(taskInstanceDao.queryById(10)).thenReturn(taskInstance); + String result = loggerService.queryLog(loginUser, projectCode, 10, 1, 1); + assertEquals("log content", result); + + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); } @Test public void testGetLogBytesInSpecifiedProject() { long projectCode = 1L; when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Project project = getProject(projectCode); User loginUser = new User(); loginUser.setId(-1); @@ -272,9 +315,24 @@ public class LoggerServiceTest { taskInstance.setHost("127.0.0.1:" + nettyServerPort); taskInstance.setLogPath("/temp/log"); doNothing().when(projectService).checkProjectAndAuthThrowException(loginUser, projectCode, DOWNLOAD_LOG); + + when(taskInstanceDao.queryById(1)).thenReturn(null); + assertThrowsServiceException( + Status.INTERNAL_SERVER_ERROR_ARGS, () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(2L); + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(1L); + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR, + () -> loggerService.getLogBytes(loginUser, projectCode, 100)); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java index 0dc71841b4..f33571d309 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.utils; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.User; + import java.nio.charset.StandardCharsets; import java.util.Random; @@ -27,4 +30,19 @@ public class ServiceTestUtil { new Random().nextBytes(bitArray); return new String(bitArray, StandardCharsets.UTF_8); } + + private static User getUser(Integer userId, String userName, UserType userType) { + User user = new User(); + user.setUserType(userType); + user.setId(userId); + user.setUserName(userName); + return user; + } + + public static User getAdminUser() { + return getUser(1, "admin", UserType.ADMIN_USER); + } + public static User getGeneralUser() { + return getUser(10, "user", UserType.GENERAL_USER); + } } From 32f92889746ccff0ee0379fd1b17ebe49a0591a7 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 19 Apr 2024 10:31:34 +0800 Subject: [PATCH 61/88] Fix dsip name (#15876) --- .github/ISSUE_TEMPLATE/dsip-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml index 0908bba9e7..f54421b066 100644 --- a/.github/ISSUE_TEMPLATE/dsip-request.yml +++ b/.github/ISSUE_TEMPLATE/dsip-request.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: Feature request +name: DSIP description: Suggest an idea for this project title: "[DSIP-][Module Name] DSIP title" labels: [ "DSIP", "Waiting for reply" ] From d306f1d04b3b4259093d139a28b2ad4d55d1dcf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Fri, 19 Apr 2024 15:20:31 +0800 Subject: [PATCH 62/88] Refactor record audit log logic (#15881) --- .../api/audit/OperatorLogAspect.java | 74 +++++++++++++++---- .../api/audit/OperatorUtils.java | 25 ++----- .../api/audit/operator/AuditOperator.java | 16 ++-- .../api/audit/operator/BaseAuditOperator.java | 43 +++++------ .../impl/TaskDefinitionServiceImpl.java | 6 +- .../common/constants/Constants.java | 4 + .../common/enums/AuditModelType.java | 2 +- 7 files changed, 100 insertions(+), 70 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java index da8a7bd3e6..aaf35d5b66 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java @@ -17,18 +17,26 @@ package org.apache.dolphinscheduler.api.audit; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; -import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.lang.reflect.Method; +import java.util.List; import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.AfterReturning; +import org.aspectj.lang.annotation.AfterThrowing; import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; import org.aspectj.lang.annotation.Pointcut; import org.aspectj.lang.reflect.MethodSignature; import org.springframework.stereotype.Component; @@ -40,34 +48,74 @@ import io.swagger.v3.oas.annotations.Operation; @Component public class OperatorLogAspect { + private static final ThreadLocal auditThreadLocal = new ThreadLocal<>(); + @Pointcut("@annotation(org.apache.dolphinscheduler.api.audit.OperatorLog)") public void logPointCut() { } - @Around("logPointCut()") - public Object around(ProceedingJoinPoint point) throws Throwable { + @Before("logPointCut()") + public void before(JoinPoint point) { MethodSignature signature = (MethodSignature) point.getSignature(); Method method = signature.getMethod(); - OperatorLog operatorLog = method.getAnnotation(OperatorLog.class); - Operation operation = method.getAnnotation(Operation.class); + if (operation == null) { log.warn("Operation is null of method: {}", method.getName()); - return point.proceed(); + return; } - long beginTime = System.currentTimeMillis(); Map paramsMap = OperatorUtils.getParamsMap(point, signature); - Result result = (Result) point.proceed(); + User user = OperatorUtils.getUser(paramsMap); + if (user == null) { + log.error("user is null"); + return; + } + + AuditType auditType = operatorLog.auditType(); + try { AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); - long latency = System.currentTimeMillis() - beginTime; - operator.recordAudit(paramsMap, result, latency, operation, operatorLog); + List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); + operator.setRequestParam(auditType, auditLogList, paramsMap); + AuditContext auditContext = + new AuditContext(auditLogList, paramsMap, operatorLog, System.currentTimeMillis(), operator); + auditThreadLocal.set(auditContext); } catch (Throwable throwable) { log.error("Record audit log error", throwable); } + } - return result; + @AfterReturning(value = "logPointCut()", returning = "returnValue") + public void afterReturning(Object returnValue) { + try { + AuditContext auditContext = auditThreadLocal.get(); + if (auditContext == null) { + return; + } + auditContext.getOperator().recordAudit(auditContext, returnValue); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } finally { + auditThreadLocal.remove(); + } + } + + @AfterThrowing("logPointCut()") + public void afterThrowing() { + auditThreadLocal.remove(); + } + + @Getter + @Setter + @AllArgsConstructor + public static class AuditContext { + + List auditLogList; + Map paramsMap; + OperatorLog operatorLog; + long beginTime; + AuditOperator operator; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index 10ccf8d648..8dc628b576 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.audit; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuditModelType; import org.apache.dolphinscheduler.common.enums.AuditOperationType; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -36,24 +37,12 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.JoinPoint; import org.aspectj.lang.reflect.MethodSignature; @Slf4j public class OperatorUtils { - protected void changeObjectForVersionRelated(AuditOperationType auditOperationType, Map paramsMap, - List auditLogList) { - switch (auditOperationType) { - case SWITCH_VERSION: - case DELETE_VERSION: - auditLogList.get(0).setModelName(paramsMap.get("version").toString()); - break; - default: - break; - } - } - public static boolean resultFail(Result result) { return result != null && result.isFailed(); } @@ -66,7 +55,7 @@ public class OperatorUtils { auditLog.setOperationType(auditType.getAuditOperationType().getName()); auditLog.setDescription(apiDescription); auditLog.setCreateTime(new Date()); - + auditLogList.add(auditLog); return auditLogList; } @@ -80,7 +69,7 @@ public class OperatorUtils { return null; } - public static Map getParamsMap(ProceedingJoinPoint point, MethodSignature signature) { + public static Map getParamsMap(JoinPoint point, MethodSignature signature) { Object[] args = point.getArgs(); String[] strings = signature.getParameterNames(); @@ -95,7 +84,7 @@ public class OperatorUtils { public static AuditOperationType modifyReleaseOperationType(AuditType auditType, Map paramsMap) { switch (auditType.getAuditOperationType()) { case RELEASE: - ReleaseState releaseState = (ReleaseState) paramsMap.get("releaseState"); + ReleaseState releaseState = (ReleaseState) paramsMap.get(Constants.RELEASE_STATE); if (releaseState == null) { break; } @@ -109,7 +98,7 @@ public class OperatorUtils { } break; case EXECUTE: - ExecuteType executeType = (ExecuteType) paramsMap.get("executeType"); + ExecuteType executeType = (ExecuteType) paramsMap.get(Constants.EXECUTE_TYPE); if (executeType == null) { break; } @@ -184,7 +173,7 @@ public class OperatorUtils { } public static boolean isUdfResource(Map paramsMap) { - ResourceType resourceType = (ResourceType) paramsMap.get("type"); + ResourceType resourceType = (ResourceType) paramsMap.get(Constants.STRING_PLUGIN_PARAM_TYPE); return resourceType != null && resourceType.equals(ResourceType.UDF); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java index 7d76d5c4bd..c3a9a845f5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java @@ -17,18 +17,16 @@ package org.apache.dolphinscheduler.api.audit.operator; -import org.apache.dolphinscheduler.api.audit.OperatorLog; -import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import java.util.List; import java.util.Map; -import io.swagger.v3.oas.annotations.Operation; - public interface AuditOperator { - void recordAudit(Map paramsMap, - Result result, - long latency, - Operation operation, - OperatorLog operatorLog) throws Throwable; + void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue); + + void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java index 270a5e4df4..0ab607da65 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.api.audit.operator; import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; import org.apache.dolphinscheduler.api.audit.OperatorUtils; import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.service.AuditService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.commons.lang3.math.NumberUtils; @@ -36,7 +36,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.base.Strings; -import io.swagger.v3.oas.annotations.Operation; @Service @Slf4j @@ -46,40 +45,34 @@ public abstract class BaseAuditOperator implements AuditOperator { private AuditService auditService; @Override - public void recordAudit(Map paramsMap, - Result result, - long latency, - Operation operation, - OperatorLog operatorLog) { + public void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue) { + Result result = new Result<>(); + if (returnValue instanceof Result) { + result = (Result) returnValue; + if (OperatorUtils.resultFail(result)) { + log.error("request fail, code {}", result.getCode()); + return; + } + } + + long latency = System.currentTimeMillis() - auditContext.getBeginTime(); + List auditLogList = auditContext.getAuditLogList(); + + Map paramsMap = auditContext.getParamsMap(); + OperatorLog operatorLog = auditContext.getOperatorLog(); AuditType auditType = operatorLog.auditType(); - User user = OperatorUtils.getUser(paramsMap); - - if (user == null) { - log.error("user is null"); - return; - } - - List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); - setRequestParam(auditType, auditLogList, paramsMap); - - if (OperatorUtils.resultFail(result)) { - log.error("request fail, code {}", result.getCode()); - return; - } - setObjectIdentityFromReturnObject(auditType, result, auditLogList); - modifyAuditOperationType(auditType, paramsMap, auditLogList); modifyAuditObjectType(auditType, paramsMap, auditLogList); auditLogList.forEach(auditLog -> auditLog.setLatency(latency)); auditLogList.forEach(auditLog -> auditService.addAudit(auditLog)); - } - protected void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { + @Override + public void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { String[] paramNameArr = auditType.getRequestParamName(); if (paramNameArr.length == 0) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 8b01df1319..71e4d04d6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -105,8 +105,6 @@ import com.google.common.collect.Lists; @Slf4j public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDefinitionService { - private static final String RELEASESTATE = "releaseState"; - @Autowired private ProjectMapper projectMapper; @@ -1297,7 +1295,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } if (null == releaseState) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(code); @@ -1337,7 +1335,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe break; default: log.warn("Parameter releaseState is invalid."); - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } int update = taskDefinitionMapper.updateById(taskDefinition); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 19e1a1fabb..b5bbf740e9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -747,4 +747,8 @@ public final class Constants { * K8S sensitive param */ public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : \"))).*?(?=(\",\\n))"; + + public static final String RELEASE_STATE = "releaseState"; + public static final String EXECUTE_TYPE = "executeType"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java index 449f046f4a..5f046882c1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java @@ -29,7 +29,7 @@ import lombok.Getter; @Getter public enum AuditModelType { - PROJECT("Project", null), // 1 + PROJECT("Project", null), PROCESS("Process", PROJECT), PROCESS_INSTANCE("ProcessInstance", PROCESS), TASK("Task", PROCESS), From a9decc911f8141b7dbc1de8b4f0599e0e170a1ff Mon Sep 17 00:00:00 2001 From: Gallardot Date: Fri, 19 Apr 2024 16:52:17 +0800 Subject: [PATCH 63/88] [Bug][Helm] fix image.registry (#15860) Signed-off-by: Gallardot Co-authored-by: fuchanghai Co-authored-by: Rick Cheng --- deploy/kubernetes/dolphinscheduler/README.md | 2 +- .../kubernetes/dolphinscheduler/values.yaml | 2 +- docs/docs/en/guide/installation/kubernetes.md | 19 +++++++++---------- docs/docs/zh/guide/installation/kubernetes.md | 19 +++++++++---------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 33633f3b2e..ba533b6e47 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -174,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | image.master | string | `"dolphinscheduler-master"` | master image | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent | | image.pullSecret | string | `""` | Specify a imagePullSecrets | -| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler | +| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler | | image.tag | string | `"latest"` | Docker image version for the DolphinScheduler | | image.tools | string | `"dolphinscheduler-tools"` | tools image | | image.worker | string | `"dolphinscheduler-worker"` | worker image | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 98c2f70db0..7a04ff5604 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -31,7 +31,7 @@ initImage: image: # -- Docker image repository for the DolphinScheduler - registry: apache/dolphinscheduler + registry: apache # -- Docker image version for the DolphinScheduler tag: latest # -- Image pull policy. Options: Always, Never, IfNotPresent diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index 8e58fdeccd..a6587a5855 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco ## Install DolphinScheduler -Please download the source code package `apache-dolphinscheduler--src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download) - -To publish the release name `dolphinscheduler` version, please execute the following commands: - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# Choose the corresponding version yourself +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` To publish the release name `dolphinscheduler` version to `test` namespace: diff --git a/docs/docs/zh/guide/installation/kubernetes.md b/docs/docs/zh/guide/installation/kubernetes.md index 20cd8907e8..f4b95de27b 100644 --- a/docs/docs/zh/guide/installation/kubernetes.md +++ b/docs/docs/zh/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务 ## 安装 dolphinscheduler -请下载源码包 apache-dolphinscheduler--src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download) - -发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令： - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# 自行选择对应的版本 +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` 将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中： From 285c5a8eb541f46b6e7b508729ba4b5178d09152 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 19 Apr 2024 18:12:40 +0800 Subject: [PATCH 64/88] [DSIP-28] Donnot scan whole bean under classpath (#15874) --- .../dolphinscheduler/alert/AlertServer.java | 13 +- .../api/ApiApplicationServer.java | 26 +- .../api/dto/task/TaskUpdateRequest.java | 5 +- .../impl/ProcessDefinitionServiceImpl.java | 7 +- .../impl/ProcessInstanceServiceImpl.java | 13 +- .../impl/TaskDefinitionServiceImpl.java | 75 ++- .../service/ProcessInstanceServiceTest.java | 36 +- .../TaskDefinitionServiceImplTest.java | 460 +++++++++--------- .../src/test/resources/logback-spring.xml | 57 +++ .../common/CommonConfiguration.java | 26 + .../common/log/remote/RemoteLogUtils.java | 1 - .../common/process/HttpProperty.java | 124 ----- .../api/DatabaseEnvironmentCondition.java | 39 ++ ...java => H2DaoPluginAutoConfiguration.java} | 8 +- .../h2/H2DatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + ...a => MysqlDaoPluginAutoConfiguration.java} | 8 +- .../MysqlDatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + ...PostgresqlDaoPluginAutoConfiguration.java} | 8 +- ...ostgresqlDatabaseEnvironmentCondition.java | 28 ++ .../main/resources/META-INF/spring.factories | 19 + .../dao/DaoConfiguration.java | 2 +- .../dolphinscheduler/dao/BaseDaoTest.java | 2 - .../dao/repository/impl/AlertDaoTest.java | 2 - .../server/master/MasterServer.java | 28 +- .../runner/TaskExecutionContextFactory.java | 7 +- .../src/test/resources/logback.xml | 2 +- ...ation.java => MeterAutoConfiguration.java} | 13 +- .../meter/metrics/DefaultMetricsProvider.java | 3 - .../main/resources/META-INF/spring.factories | 19 + .../registry/api/RegistryConfiguration.java | 33 ++ .../plugin/registry/etcd/EtcdRegistry.java | 12 +- .../etcd/EtcdRegistryAutoConfiguration.java | 48 ++ .../registry/etcd/EtcdRegistryProperties.java | 2 - .../main/resources/META-INF/spring.factories | 19 + .../jdbc/JdbcRegistryAutoConfiguration.java | 2 + .../registry/zookeeper/ZookeeperRegistry.java | 7 +- .../ZookeeperRegistryAutoConfiguration.java | 46 ++ .../ZookeeperRegistryProperties.java | 94 +--- .../main/resources/META-INF/spring.factories | 19 + ... => QuartzSchedulerAutoConfiguration.java} | 2 +- .../main/resources/META-INF/spring.factories | 19 + .../service/ServiceConfiguration.java | 26 + .../service/process/ProcessServiceImpl.java | 4 - .../service/utils/CommonUtils.java | 108 ---- .../service/process/ProcessServiceTest.java | 5 - .../service/utils/CommonUtilsTest.java | 71 --- .../dolphinscheduler/StandaloneServer.java | 2 +- .../src/main/resources/application.yaml | 2 +- .../plugin/task/api/TaskPluginManager.java | 27 +- .../datasource/UpgradeDolphinScheduler.java | 4 +- .../server/worker/WorkerServer.java | 22 +- .../runner/DefaultWorkerTaskExecutor.java | 3 - .../DefaultWorkerTaskExecutorFactory.java | 5 - .../worker/runner/WorkerTaskExecutor.java | 5 +- .../WorkerTaskExecutorFactoryBuilder.java | 11 - .../runner/DefaultWorkerTaskExecutorTest.java | 5 - .../WorkerTaskExecutorThreadPoolTest.java | 3 +- .../TaskInstanceOperationFunctionTest.java | 5 - .../src/test/resources/logback.xml | 14 +- 61 files changed, 895 insertions(+), 855 deletions(-) create mode 100644 dolphinscheduler-api/src/test/resources/logback-spring.xml create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/{H2DaoPluginConfiguration.java => H2DaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/{MysqlDaoPluginConfiguration.java => MysqlDaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories rename dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/{PostgresqlDaoPluginConfiguration.java => PostgresqlDaoPluginAutoConfiguration.java} (88%) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories rename dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/{MeterConfiguration.java => MeterAutoConfiguration.java} (87%) create mode 100644 dolphinscheduler-meter/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories rename dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/{QuartzSchedulerConfiguration.java => QuartzSchedulerAutoConfiguration.java} (97%) create mode 100644 dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories create mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java delete mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java delete mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ff0088ea8d..55c5c3446c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -23,11 +23,13 @@ import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -37,14 +39,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; @Slf4j +@Import({CommonConfiguration.class, + DaoConfiguration.class, + RegistryConfiguration.class}) @SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) public class AlertServer { @Autowired diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 20a6412076..6f7d8f43d2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -18,13 +18,17 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; +import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -38,21 +42,19 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.servlet.ServletComponentScan; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; import org.springframework.context.event.EventListener; +@Slf4j +@Import({DaoConfiguration.class, + CommonConfiguration.class, + ServiceConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) @ServletComponentScan @SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) -@Slf4j public class ApiApplicationServer { - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private PluginDao pluginDao; @@ -66,8 +68,8 @@ public class ApiApplicationServer { public void run(ApplicationReadyEvent readyEvent) { log.info("Received spring application context ready event will load taskPlugin and write to DB"); // install task plugin - taskPluginManager.loadPlugin(); - for (Map.Entry entry : taskPluginManager.getTaskChannelFactoryMap().entrySet()) { + TaskPluginManager.loadPlugin(); + for (Map.Entry entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { String taskPluginName = entry.getKey(); TaskChannelFactory taskChannelFactory = entry.getValue(); List params = taskChannelFactory.getParams(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java index b5a8ea7a46..d7b026ed16 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java @@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.commons.beanutils.BeanUtils; -import java.lang.reflect.InvocationTargetException; import java.util.Date; import lombok.Data; +import lombok.SneakyThrows; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -107,7 +107,8 @@ public class TaskUpdateRequest { * @param taskDefinition exists task definition object * @return task definition */ - public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) throws InvocationTargetException, IllegalAccessException, InstantiationException, NoSuchMethodException { + @SneakyThrows + public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) { TaskDefinition taskDefinitionDeepCopy = (TaskDefinition) BeanUtils.cloneBean(taskDefinition); assert taskDefinitionDeepCopy != null; if (this.name != null) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 2bf84a0bcc..e6893b04fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -238,9 +238,6 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro @Autowired private DataSourceMapper dataSourceMapper; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private WorkFlowLineageService workFlowLineageService; @@ -424,7 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson); } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -1618,7 +1615,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check whether the process definition json is normal for (TaskNode taskNode : taskNodes) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskNode.getType()) .taskParams(taskNode.getTaskParams()) .dependence(taskNode.getDependence()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 36cc986607..06e9fd95db 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -70,11 +70,9 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.RelationSubWorkflowMapper; -import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; @@ -177,18 +175,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce @Autowired UsersService usersService; - @Autowired - private TenantMapper tenantMapper; - @Autowired TaskDefinitionMapper taskDefinitionMapper; - @Autowired - private TaskPluginManager taskPluginManager; - - @Autowired - private ScheduleMapper scheduleMapper; - @Autowired private RelationSubWorkflowMapper relationSubWorkflowMapper; @@ -725,7 +714,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 71e4d04d6f..2887606a1d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -75,7 +75,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -135,9 +134,6 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe @Autowired private ProcessService processService; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private ProcessDefinitionService processDefinitionService; @@ -147,8 +143,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create task definition * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param taskDefinitionJson task definition json */ @Transactional @@ -171,7 +167,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -212,7 +208,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe Project project = projectMapper.queryByCode(taskDefinition.getProjectCode()); projectService.checkProjectAndAuthThrowException(user, project, permissions); - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -242,7 +238,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Create resource task definition * - * @param loginUser login user + * @param loginUser login user * @param taskCreateRequest task definition json * @return new TaskDefinition have created */ @@ -286,11 +282,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create single task definition that binds the workflow * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param processDefinitionCode process definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return create result code */ @Transactional @@ -325,7 +321,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return result; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -412,10 +408,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * query task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code * @param processCode process code - * @param taskName task name + * @param taskName task name */ @Override public Map queryTaskDefinitionByName(User loginUser, long projectCode, long processCode, @@ -473,12 +469,12 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Delete resource task definition by code - * + *

* Only task release state offline and no downstream tasks can be deleted, will also remove the exists * task relation [upstreamTaskCode, taskCode] * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code */ @Transactional @Override @@ -546,9 +542,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task code * @param taskDefinitionJsonObj task definition json object */ @Transactional @@ -604,8 +600,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param taskCode task code + * @param loginUser login user + * @param taskCode task code * @param taskUpdateRequest task definition json object * @return new TaskDefinition have updated */ @@ -619,13 +615,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe throw new ServiceException(Status.TASK_DEFINITION_NOT_EXISTS, taskCode); } - TaskDefinition taskDefinitionUpdate; - try { - taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); - } catch (InvocationTargetException | IllegalAccessException | InstantiationException - | NoSuchMethodException e) { - throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, taskUpdateRequest.toString()); - } + TaskDefinition taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); + this.checkTaskDefinitionValid(loginUser, taskDefinitionUpdate, TASK_DEFINITION_UPDATE); this.TaskDefinitionUpdateValid(taskDefinitionOriginal, taskDefinitionUpdate); @@ -656,7 +647,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe * Get resource task definition by code * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code * @return TaskDefinition */ @Override @@ -674,7 +665,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Get resource task definition according to query parameter * - * @param loginUser login user + * @param loginUser login user * @param taskFilterRequest taskFilterRequest object you want to filter the resource task definitions * @return TaskDefinitions of page */ @@ -741,7 +732,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return null; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionToUpdate.getTaskType()) .taskParams(taskDefinitionToUpdate.getTaskParams()) .dependence(taskDefinitionToUpdate.getDependence()) @@ -846,11 +837,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition and upstream * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task definition code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return update result code */ @Override @@ -1019,10 +1010,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * switch task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code - * @param taskCode task code - * @param version the version user want to switch + * @param taskCode task code + * @param version the version user want to switch */ @Transactional @Override @@ -1277,9 +1268,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * release task definition * - * @param loginUser login user - * @param projectCode project code - * @param code task definition code + * @param loginUser login user + * @param projectCode project code + * @param code task definition code * @param releaseState releaseState * @return update result code */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 9fb4d83052..7d9a4f9338 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -84,6 +84,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -142,9 +143,6 @@ public class ProcessInstanceServiceTest { @Mock TaskDefinitionMapper taskDefinitionMapper; - @Mock - TaskPluginManager taskPluginManager; - @Mock ScheduleMapper scheduleMapper; @@ -625,21 +623,27 @@ public class ProcessInstanceServiceTest { List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result); putMsg(result, Status.SUCCESS, projectCode); - when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Map processInstanceFinishRes = - processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - // success - when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); - putMsg(result, Status.SUCCESS, projectCode); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Map processInstanceFinishRes = + processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) - .thenReturn(1); - Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + // success + when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); + putMsg(result, Status.SUCCESS, projectCode); + + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) + .thenReturn(1); + Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java index e77eab8029..ff8ca6ba2c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java @@ -17,13 +17,19 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_UPDATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest; import org.apache.dolphinscheduler.api.dto.task.TaskUpdateRequest; @@ -75,13 +81,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class TaskDefinitionServiceImplTest { @InjectMocks @@ -114,9 +124,6 @@ public class TaskDefinitionServiceImplTest { @Mock private ProcessTaskRelationMapper processTaskRelationMapper; - @Mock - private TaskPluginManager taskPluginManager; - @Mock private ProcessTaskRelationService processTaskRelationService; @@ -155,61 +162,73 @@ public class TaskDefinitionServiceImplTest { @Test public void createTaskDefinition() { - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, result)) - .thenReturn(true); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); + Map result = new HashMap<>(); + when(projectService.hasProjectAndWritePerm(user, project, result)) + .thenReturn(true); - String createTaskDefinitionJson = - "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" - + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," - + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" - + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," - + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," - + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," - + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; - Map relation = taskDefinitionService - .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + String createTaskDefinitionJson = + "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" + + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," + + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" + + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," + + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," + + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," + + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; + Map relation = taskDefinitionService + .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + + } } @Test public void updateTaskDefinition() { - String taskDefinitionJson = getTaskDefinitionJson();; + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS, PROJECT_CODE); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); - Mockito.when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); - Mockito.when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) - .thenReturn(getProcessTaskRelationList2()); - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); - result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - // failure - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson)); - Assertions.assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), - ((ServiceException) exception).getCode()); + when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); + when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); + when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); + when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); + when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) + .thenReturn(getProcessTaskRelationList2()); + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); + result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); + assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + // failure + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); + exception = Assertions.assertThrows(ServiceException.class, + () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson)); + assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), + ((ServiceException) exception).getCode()); + } } @@ -217,72 +236,72 @@ public class TaskDefinitionServiceImplTest { public void queryTaskDefinitionByName() { String taskName = "task"; Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(result); - Mockito.when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) + when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) .thenReturn(new TaskDefinition()); Map relation = taskDefinitionService .queryTaskDefinitionByName(user, PROJECT_CODE, PROCESS_DEFINITION_CODE, taskName); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } @Test public void deleteTaskDefinitionByCode() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); // error task definition not find exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error delete single task definition object - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), + assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_DELETE); - Mockito.when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); Assertions.assertDoesNotThrow(() -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); } @Test public void switchVersion() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when( + when( projectService.checkProjectAndAuth(user, project, PROJECT_CODE, WORKFLOW_SWITCH_TO_THIS_VERSION)) - .thenReturn(result); + .thenReturn(result); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) .thenReturn(new TaskDefinitionLog()); TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(PROJECT_CODE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)) + when(taskDefinitionMapper.queryByCode(TASK_CODE)) .thenReturn(taskDefinition); - Mockito.when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); + when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); Map relation = taskDefinitionService .switchVersion(user, PROJECT_CODE, TASK_CODE, VERSION); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } private void putMsg(Map result, Status status, Object... statusParams) { @@ -331,7 +350,7 @@ public class TaskDefinitionServiceImplTest { @Test public void genTaskCodeList() { Map genTaskCodeList = taskDefinitionService.genTaskCodeList(10); - Assertions.assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); } @Test @@ -348,31 +367,31 @@ public class TaskDefinitionServiceImplTest { taskMainInfo.setUpstreamTaskName("4"); taskMainInfoIPage.setRecords(Collections.singletonList(taskMainInfo)); taskMainInfoIPage.setTotal(10L); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(checkResult); - Mockito.when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), + when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), Mockito.isNull(), Mockito.anyString(), Mockito.isNull())) - .thenReturn(taskMainInfoIPage); - Mockito.when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) + .thenReturn(taskMainInfoIPage); + when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) .thenReturn(Collections.singletonList(taskMainInfo)); Result result = taskDefinitionService.queryTaskDefinitionListPaging(user, PROJECT_CODE, null, null, null, pageNo, pageSize); - Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test public void testReleaseTaskDefinition() { - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); Project project = getProject(); // check task dose not exist Map result = new HashMap<>(); putMsg(result, Status.TASK_DEFINE_NOT_EXIST, TASK_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); Map map = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); + assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); // process definition offline putMsg(result, Status.SUCCESS); @@ -384,23 +403,23 @@ public class TaskDefinitionServiceImplTest { "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}}"; taskDefinition.setTaskParams(params); taskDefinition.setTaskType("SHELL"); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(taskDefinition); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) .thenReturn(taskDefinitionLog); Map offlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); // process definition online, resource exist Map onlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.ONLINE); - Assertions.assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); // release error code Map failResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.getEnum(2)); - Assertions.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); + assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); } @Test @@ -410,133 +429,127 @@ public class TaskDefinitionServiceImplTest { taskCreateRequest.setWorkflowCode(PROCESS_DEFINITION_CODE); // error process definition not find - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROCESS_DEFINE_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition taskCreateRequest.setTaskParams(TASK_PARAMETER); - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); - // error create task definition object - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); - // error sync to task definition log - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error create task definition object + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Mockito.when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), - isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); - Assertions.assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + // error sync to task definition log + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), + isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); + assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + } } @Test public void testUpdateTaskDefinitionV2() { TaskUpdateRequest taskUpdateRequest = new TaskUpdateRequest(); + TaskDefinition taskDefinition = getTaskDefinition(); + Project project = getProject(); // error task definition not exists - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_EXISTS, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_EXISTS.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(isA(Long.class))).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) - .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(projectMapper.queryByCode(isA(Long.class))).thenReturn(project); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + .checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); - // error task definition already online - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(false); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + } - // error task definition nothing update - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_CHANGE.getCode(), ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + // error task definition nothing update + when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_CHANGE, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition version invalid - taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.DATA_IS_NOT_VALID.getCode(), ((ServiceException) exception).getCode()); + // error task definition version invalid + taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); + assertThrowsServiceException(Status.DATA_IS_NOT_VALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition update effect number - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.UPDATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition update effect number + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.UPDATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition log insert - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition log insert + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Assertions.assertDoesNotThrow( - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + Assertions.assertDoesNotThrow( + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - TaskDefinition taskDefinition = - taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); - Assertions.assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + taskDefinition = + taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); + assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + } } @Test @@ -549,28 +562,28 @@ public class TaskDefinitionServiceImplTest { ArrayList taskDefinitionLogs = new ArrayList<>(); taskDefinitionLogs.add(taskDefinitionLog); Integer version = 1; - Mockito.when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); + when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); // saveProcessDefine - Mockito.when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); - Mockito.when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); + when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); int insertVersion = processServiceImpl.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE); - Mockito.when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) .thenReturn(insertVersion); - Assertions.assertEquals(insertVersion, version + 1); + assertEquals(insertVersion, version + 1); // saveTaskRelation List processTaskRelationLogList = getProcessTaskRelationLogList(); - Mockito.when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) + when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) .thenReturn(processTaskRelationList); - Mockito.when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); - Mockito.when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); int insertResult = processServiceImpl.saveTaskRelation(loginUser, processDefinition.getProjectCode(), processDefinition.getCode(), insertVersion, processTaskRelationLogList, taskDefinitionLogs, Boolean.TRUE); - Assertions.assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); + assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); Assertions.assertDoesNotThrow( () -> taskDefinitionService.updateDag(loginUser, processDefinition.getCode(), processTaskRelationList, taskDefinitionLogs)); @@ -581,55 +594,60 @@ public class TaskDefinitionServiceImplTest { // error task definition not exists exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition not exists - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), + assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); Assertions.assertDoesNotThrow(() -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); } @Test public void testUpdateTaskWithUpstream() { + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); + TaskDefinition taskDefinition = getTaskDefinition(); + taskDefinition.setFlag(Flag.NO); + TaskDefinition taskDefinitionSecond = getTaskDefinition(); + taskDefinitionSecond.setCode(5); - String taskDefinitionJson = getTaskDefinitionJson(); - TaskDefinition taskDefinition = getTaskDefinition(); - taskDefinition.setFlag(Flag.NO); - TaskDefinition taskDefinitionSecond = getTaskDefinition(); - taskDefinitionSecond.setCode(5); + user.setUserType(UserType.ADMIN_USER); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); - user.setUserType(UserType.ADMIN_USER); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); + when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) + .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - Mockito.when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) - .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - - Mockito.when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) - .thenReturn(getProcessTaskRelationListV2()); - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); - Mockito.when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); - // success - Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, - taskDefinitionJson, UPSTREAM_CODE); - Assertions.assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); - user.setUserType(UserType.GENERAL_USER); + when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) + .thenReturn(getProcessTaskRelationListV2()); + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)) + .thenReturn(getProcessDefinition()); + when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); + when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); + when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); + // success + Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson, UPSTREAM_CODE); + assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); + user.setUserType(UserType.GENERAL_USER); + } } private String getTaskDefinitionJson() { diff --git a/dolphinscheduler-api/src/test/resources/logback-spring.xml b/dolphinscheduler-api/src/test/resources/logback-spring.xml new file mode 100644 index 0000000000..9159c3b02d --- /dev/null +++ b/dolphinscheduler-api/src/test/resources/logback-spring.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-api.log + + ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + 50GB + true + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java new file mode 100644 index 0000000000..5411e4cbc1 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ComponentScan("org.apache.dolphinscheduler.common") +public class CommonConfiguration { +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java index 25d8024474..8aab70f304 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java @@ -40,7 +40,6 @@ public class RemoteLogUtils { @Autowired private RemoteLogService autowiredRemoteLogService; - @PostConstruct private void init() { remoteLogService = autowiredRemoteLogService; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java deleted file mode 100644 index 11786fd5a3..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.process; - -import org.apache.dolphinscheduler.common.enums.HttpParametersType; - -import java.util.Objects; - -public class HttpProperty { - - /** - * key - */ - private String prop; - - /** - * httpParametersType - */ - private HttpParametersType httpParametersType; - - /** - * value - */ - private String value; - - public HttpProperty() { - } - - public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { - this.prop = prop; - this.httpParametersType = httpParametersType; - this.value = value; - } - - /** - * getter method - * - * @return the prop - * @see HttpProperty#prop - */ - public String getProp() { - return prop; - } - - /** - * setter method - * - * @param prop the prop to set - * @see HttpProperty#prop - */ - public void setProp(String prop) { - this.prop = prop; - } - - /** - * getter method - * - * @return the value - * @see HttpProperty#value - */ - public String getValue() { - return value; - } - - /** - * setter method - * - * @param value the value to set - * @see HttpProperty#value - */ - public void setValue(String value) { - this.value = value; - } - - public HttpParametersType getHttpParametersType() { - return httpParametersType; - } - - public void setHttpParametersType(HttpParametersType httpParametersType) { - this.httpParametersType = httpParametersType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - HttpProperty property = (HttpProperty) o; - return Objects.equals(prop, property.prop) - && Objects.equals(value, property.value); - } - - @Override - public int hashCode() { - return Objects.hash(prop, value); - } - - @Override - public String toString() { - return "HttpProperty{" - + "prop='" + prop + '\'' - + ", httpParametersType=" + httpParametersType - + ", value='" + value + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..32fb807a9b --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.api; + +import java.util.Arrays; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class DatabaseEnvironmentCondition implements Condition { + + private final String profile; + + public DatabaseEnvironmentCondition(String profile) { + this.profile = profile; + } + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + String[] activeProfiles = context.getEnvironment().getActiveProfiles(); + return Arrays.asList(activeProfiles).contains(profile); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java index 9aea94f77d..f496679ed5 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("h2") -@Configuration -public class H2DaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(H2DatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class H2DaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..894f38bd20 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.h2; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class H2DatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public H2DatabaseEnvironmentCondition() { + super("h2"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..c899dfb43f --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.h2.H2DaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java index 8b37fca67b..5fb3a350ae 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java @@ -28,14 +28,14 @@ import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("mysql") -@Configuration -public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Configuration(proxyBeanMethods = false) +@Conditional(MysqlDatabaseEnvironmentCondition.class) +public class MysqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2136e1354e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.mysql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class MysqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public MysqlDatabaseEnvironmentCondition() { + super("mysql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..386c80c676 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.mysql.MysqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java index e57c84fab9..f0467bfd07 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonit import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("postgresql") -@Configuration -public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(PostgresqlDatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class PostgresqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2c71ea8442 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.postgresql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class PostgresqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public PostgresqlDatabaseEnvironmentCondition() { + super("postgresql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..fd6a5f07b9 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.postgresql.PostgresqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index 985f5b56b4..e089c8086b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -40,8 +40,8 @@ import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; @Configuration +@ComponentScan("org.apache.dolphinscheduler.dao") @EnableAutoConfiguration -@ComponentScan({"org.apache.dolphinscheduler.dao.plugin"}) @MapperScan(basePackages = "org.apache.dolphinscheduler.dao.mapper", sqlSessionFactoryRef = "sqlSessionFactory") public class DaoConfiguration { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java index 231c947f65..6f14eb436e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java @@ -22,7 +22,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; -import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; @ExtendWith(MockitoExtension.class) @@ -30,6 +29,5 @@ import org.springframework.transaction.annotation.Transactional; @SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) @Transactional @Rollback -@EnableTransactionManagement public abstract class BaseDaoTest { } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index c0a841c1a0..fe8545f3e2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -34,7 +34,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ActiveProfiles; -import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; @ActiveProfiles(ProfileType.H2) @@ -43,7 +42,6 @@ import org.springframework.transaction.annotation.Transactional; @SpringBootTest(classes = DaoConfiguration.class) @Transactional @Rollback -@EnableTransactionManagement public class AlertDaoTest { @Autowired diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index fd262a8c6e..7988570135 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -17,15 +17,18 @@ package org.apache.dolphinscheduler.server.master; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.scheduler.api.SchedulerApi; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient; @@ -35,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.EventExecuteService; import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap; import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator; +import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import javax.annotation.PostConstruct; @@ -45,18 +49,15 @@ import org.quartz.SchedulerException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; -@SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) -@EnableTransactionManagement -@EnableCaching @Slf4j +@Import({DaoConfiguration.class, + ServiceConfiguration.class, + CommonConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) +@SpringBootApplication public class MasterServer implements IStoppable { @Autowired @@ -65,9 +66,6 @@ public class MasterServer implements IStoppable { @Autowired private MasterRegistryClient masterRegistryClient; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private MasterSchedulerBootstrap masterSchedulerBootstrap; @@ -109,7 +107,7 @@ public class MasterServer implements IStoppable { this.masterRPCServer.start(); // install task plugin - this.taskPluginManager.loadPlugin(); + TaskPluginManager.loadPlugin(); this.masterSlotManager.start(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index ab1806ff67..b3b0cf67ea 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -90,9 +90,6 @@ public class TaskExecutionContextFactory { @Autowired private ProcessService processService; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private CuringParamsService curingParamsService; @@ -106,14 +103,14 @@ public class TaskExecutionContextFactory { ProcessInstance workflowInstance = taskInstance.getProcessInstance(); ResourceParametersHelper resources = - Optional.ofNullable(taskPluginManager.getTaskChannel(taskInstance.getTaskType())) + Optional.ofNullable(TaskPluginManager.getTaskChannel(taskInstance.getTaskType())) .map(taskChannel -> taskChannel.getResources(taskInstance.getTaskParams())) .orElse(null); setTaskResourceInfo(resources); Map businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance); - AbstractParameters baseParam = taskPluginManager.getParameters(ParametersNode.builder() + AbstractParameters baseParam = TaskPluginManager.getParameters(ParametersNode.builder() .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); Map propertyMap = curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance); diff --git a/dolphinscheduler-master/src/test/resources/logback.xml b/dolphinscheduler-master/src/test/resources/logback.xml index 4470a46391..286e35cd1f 100644 --- a/dolphinscheduler-master/src/test/resources/logback.xml +++ b/dolphinscheduler-master/src/test/resources/logback.xml @@ -65,7 +65,7 @@ - + diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java similarity index 87% rename from dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java rename to dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java index e3b140a578..fa8933d132 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterAutoConfiguration.java @@ -20,7 +20,8 @@ package org.apache.dolphinscheduler.meter; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.apache.dolphinscheduler.meter.metrics.DefaultMetricsProvider; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -42,11 +43,15 @@ import io.micrometer.core.instrument.MeterRegistry; * } * */ -@Configuration +@Configuration(proxyBeanMethods = false) @EnableAspectJAutoProxy -@EnableAutoConfiguration @ConditionalOnProperty(prefix = "metrics", name = "enabled", havingValue = "true") -public class MeterConfiguration { +public class MeterAutoConfiguration { + + @Bean + public DefaultMetricsProvider metricsProvider(MeterRegistry meterRegistry) { + return new DefaultMetricsProvider(meterRegistry); + } @Bean public TimedAspect timedAspect(MeterRegistry registry) { diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java index f1240a0541..e293e44a8d 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java @@ -19,11 +19,8 @@ package org.apache.dolphinscheduler.meter.metrics; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.springframework.stereotype.Component; - import io.micrometer.core.instrument.MeterRegistry; -@Component public class DefaultMetricsProvider implements MetricsProvider { private final MeterRegistry meterRegistry; diff --git a/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories b/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..77bc56d86e --- /dev/null +++ b/dolphinscheduler-meter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.meter.MeterAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java new file mode 100644 index 0000000000..bae3694964 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RegistryConfiguration { + + @Bean + @ConditionalOnMissingBean + public RegistryClient registryClient(Registry registry) { + return new RegistryClient(registry); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 1d1397db54..6833a6607b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -41,8 +41,6 @@ import javax.net.ssl.SSLException; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import com.google.common.base.Splitter; @@ -68,8 +66,6 @@ import io.netty.handler.ssl.SslContext; * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to * store the DolphinScheduler master/worker's metadata and do the server registry/unRegistry. */ -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @Slf4j public class EtcdRegistry implements Registry { @@ -86,6 +82,7 @@ public class EtcdRegistry implements Registry { private final Map watcherMap = new ConcurrentHashMap<>(); private static final long TIME_TO_LIVE_SECONDS = 30L; + public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException { ClientBuilder clientBuilder = Client.builder() .endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints()))) @@ -141,8 +138,7 @@ public class EtcdRegistry implements Registry { } /** - * - * @param path The prefix of the key being listened to + * @param path The prefix of the key being listened to * @param listener * @return if subcribe Returns true if no exception was thrown */ @@ -165,8 +161,8 @@ public class EtcdRegistry implements Registry { } /** - * @throws throws an exception if the unsubscribe path does not exist * @param path The prefix of the key being listened to + * @throws throws an exception if the unsubscribe path does not exist */ @Override public void unsubscribe(String path) { @@ -184,7 +180,6 @@ public class EtcdRegistry implements Registry { } /** - * * @return Returns the value corresponding to the key * @throws throws an exception if the key does not exist */ @@ -202,7 +197,6 @@ public class EtcdRegistry implements Registry { } /** - * * @param deleteOnDisconnect Does the put data disappear when the client disconnects */ @Override diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java new file mode 100644 index 0000000000..1038d312ff --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.etcd; + +import org.apache.dolphinscheduler.registry.api.Registry; + +import javax.net.ssl.SSLException; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@ComponentScan +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") +public class EtcdRegistryAutoConfiguration { + + public EtcdRegistryAutoConfiguration() { + log.info("Load EtcdRegistryAutoConfiguration"); + } + + @Bean + @ConditionalOnMissingBean(value = Registry.class) + public EtcdRegistry etcdRegistry(EtcdRegistryProperties etcdRegistryProperties) throws SSLException { + return new EtcdRegistry(etcdRegistryProperties); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index faded2a506..babb6dea76 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -21,13 +21,11 @@ import java.time.Duration; import lombok.Data; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @Data @Configuration -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @ConfigurationProperties(prefix = "registry") public class EtcdRegistryProperties { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..689817bb96 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.etcd.EtcdRegistryAutoConfiguration diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java index 09211f99fb..f21ce0d67c 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java @@ -30,6 +30,7 @@ import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration; @@ -37,6 +38,7 @@ import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import com.zaxxer.hikari.HikariDataSource; @Slf4j +@ComponentScan @Configuration(proxyBeanMethods = false) @MapperScan("org.apache.dolphinscheduler.plugin.registry.jdbc.mapper") @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 7333c10f05..3f0c3ccb59 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -50,14 +50,11 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; import lombok.NonNull; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; +import lombok.extern.slf4j.Slf4j; import com.google.common.base.Strings; -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") +@Slf4j public final class ZookeeperRegistry implements Registry { private final ZookeeperRegistryProperties.ZookeeperProperties properties; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java new file mode 100644 index 0000000000..e8fc31327e --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryAutoConfiguration.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.zookeeper; + +import org.apache.dolphinscheduler.registry.api.Registry; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@ComponentScan +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") +public class ZookeeperRegistryAutoConfiguration { + + public ZookeeperRegistryAutoConfiguration() { + log.info("Load ZookeeperRegistryAutoConfiguration"); + } + + @Bean + @ConditionalOnMissingBean(value = Registry.class) + public ZookeeperRegistry zookeeperRegistry(ZookeeperRegistryProperties zookeeperRegistryProperties) { + return new ZookeeperRegistry(zookeeperRegistryProperties); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java index 42dbc5d256..b54ebc0b1a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java @@ -19,25 +19,19 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper; import java.time.Duration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +@Data @Configuration -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") @ConfigurationProperties(prefix = "registry") public class ZookeeperRegistryProperties { private ZookeeperProperties zookeeper = new ZookeeperProperties(); - public ZookeeperProperties getZookeeper() { - return zookeeper; - } - - public void setZookeeper(ZookeeperProperties zookeeper) { - this.zookeeper = zookeeper; - } - + @Data public static final class ZookeeperProperties { private String namespace; @@ -48,91 +42,13 @@ public class ZookeeperRegistryProperties { private Duration connectionTimeout = Duration.ofSeconds(9); private Duration blockUntilConnected = Duration.ofMillis(600); - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getConnectString() { - return connectString; - } - - public void setConnectString(String connectString) { - this.connectString = connectString; - } - - public RetryPolicy getRetryPolicy() { - return retryPolicy; - } - - public void setRetryPolicy(RetryPolicy retryPolicy) { - this.retryPolicy = retryPolicy; - } - - public String getDigest() { - return digest; - } - - public void setDigest(String digest) { - this.digest = digest; - } - - public Duration getSessionTimeout() { - return sessionTimeout; - } - - public void setSessionTimeout(Duration sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } - - public Duration getConnectionTimeout() { - return connectionTimeout; - } - - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Duration getBlockUntilConnected() { - return blockUntilConnected; - } - - public void setBlockUntilConnected(Duration blockUntilConnected) { - this.blockUntilConnected = blockUntilConnected; - } - + @Data public static final class RetryPolicy { private Duration baseSleepTime = Duration.ofMillis(60); private int maxRetries; private Duration maxSleep = Duration.ofMillis(300); - public Duration getBaseSleepTime() { - return baseSleepTime; - } - - public void setBaseSleepTime(Duration baseSleepTime) { - this.baseSleepTime = baseSleepTime; - } - - public int getMaxRetries() { - return maxRetries; - } - - public void setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - } - - public Duration getMaxSleep() { - return maxSleep; - } - - public void setMaxSleep(Duration maxSleep) { - this.maxSleep = maxSleep; - } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..821f1a70c1 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperRegistryAutoConfiguration diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java similarity index 97% rename from dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java rename to dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java index 07ff8af434..34fb782587 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerConfiguration.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzSchedulerAutoConfiguration.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.Bean; @AutoConfiguration(after = {QuartzAutoConfiguration.class}) @ConditionalOnClass(value = Scheduler.class) -public class QuartzSchedulerConfiguration { +public class QuartzSchedulerAutoConfiguration { @Bean @ConditionalOnMissingBean diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..b34f896b84 --- /dev/null +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.scheduler.quartz.QuartzSchedulerAutoConfiguration diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java new file mode 100644 index 0000000000..fa831a5b6b --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/ServiceConfiguration.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.service; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@ComponentScan("org.apache.dolphinscheduler.service") +@Configuration +public class ServiceConfiguration { +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 3c207ae982..635948fc86 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -111,7 +111,6 @@ import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcCli import org.apache.dolphinscheduler.extract.common.ILogService; import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState; @@ -262,9 +261,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private WorkFlowLineageMapper workFlowLineageMapper; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private ClusterMapper clusterMapper; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java deleted file mode 100644 index e00212823a..0000000000 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.utils; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; - -import java.net.URL; -import java.nio.charset.StandardCharsets; - -import lombok.extern.slf4j.Slf4j; - -/** - * common utils - */ -@Slf4j -public class CommonUtils { - - private static final Base64 BASE64 = new Base64(); - - protected CommonUtils() { - throw new UnsupportedOperationException("Construct CommonUtils"); - } - - /** - * @return get the path of system environment variables - */ - public static String getSystemEnvPath() { - String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH); - if (StringUtils.isEmpty(envPath)) { - URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH); - - if (envDefaultPath != null) { - envPath = envDefaultPath.getPath(); - log.debug("env path :{}", envPath); - } else { - envPath = "/etc/profile"; - } - } - - return envPath; - } - - /** - * encode password - */ - public static String encodePassword(String password) { - if (StringUtils.isEmpty(password)) { - return StringUtils.EMPTY; - } - // if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false); - if (!encryptionEnable) { - return password; - } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, - DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))); - return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); - } - - /** - * decode password - */ - public static String decodePassword(String password) { - if (StringUtils.isEmpty(password)) { - return StringUtils.EMPTY; - } - - // if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false); - if (!encryptionEnable) { - return password; - } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, - DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8); - if (!passwordWithSalt.startsWith(salt)) { - log.warn("There is a password and salt mismatch: {} ", password); - return password; - } - return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8); - } - -} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index 509f9a4cf1..5c998e1f9f 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -79,7 +79,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.dp.InputType; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.service.cron.CronUtilsTest; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; @@ -102,8 +101,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * process service test @@ -112,7 +109,6 @@ import org.slf4j.LoggerFactory; @MockitoSettings(strictness = Strictness.LENIENT) public class ProcessServiceTest { - private static final Logger logger = LoggerFactory.getLogger(CronUtilsTest.class); @InjectMocks private ProcessServiceImpl processService; @Mock @@ -667,7 +663,6 @@ public class ProcessServiceTest { taskDefinition.setVersion(1); taskDefinition.setCreateTime(new Date()); taskDefinition.setUpdateTime(new Date()); - when(taskPluginManager.getParameters(any())).thenReturn(null); when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskDefinition.getCode(), taskDefinition.getVersion())).thenReturn(taskDefinition); when(taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode())).thenReturn(1); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java deleted file mode 100644 index cab280a702..0000000000 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.utils; - -import org.apache.dolphinscheduler.common.utils.FileUtils; - -import java.net.InetAddress; -import java.net.UnknownHostException; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * configuration test - */ -@ExtendWith(MockitoExtension.class) -public class CommonUtilsTest { - - private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); - - @Test - public void getSystemEnvPath() { - String envPath; - envPath = CommonUtils.getSystemEnvPath(); - Assertions.assertEquals("/etc/profile", envPath); - } - - @Test - public void getDownloadFilename() { - logger.info(FileUtils.getDownloadFilename("a.txt")); - Assertions.assertTrue(true); - } - - @Test - public void getUploadFilename() { - logger.info(FileUtils.getUploadFilename("1234", "a.txt")); - Assertions.assertTrue(true); - } - - @Test - public void test() { - InetAddress ip; - try { - ip = InetAddress.getLocalHost(); - logger.info(ip.getHostAddress()); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - Assertions.assertTrue(true); - } - -} diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java index 14808ab629..a39d3c9a22 100644 --- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java +++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java @@ -24,8 +24,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication @Slf4j +@SpringBootApplication public class StandaloneServer { public static void main(String[] args) throws Exception { diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 654113471d..5122eea2a1 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -329,4 +329,4 @@ spring: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 username: root - password: root@123 + password: root diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java index be3417466f..938aa77459 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java @@ -36,21 +36,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -@Component @Slf4j public class TaskPluginManager { - private final Map taskChannelFactoryMap = new HashMap<>(); - private final Map taskChannelMap = new HashMap<>(); + private static final Map taskChannelFactoryMap = new HashMap<>(); + private static final Map taskChannelMap = new HashMap<>(); - private final AtomicBoolean loadedFlag = new AtomicBoolean(false); + private static final AtomicBoolean loadedFlag = new AtomicBoolean(false); /** * Load task plugins from classpath. */ - public void loadPlugin() { + public static void loadPlugin() { if (!loadedFlag.compareAndSet(false, true)) { log.warn("The task plugin has already been loaded"); return; @@ -70,24 +67,24 @@ public class TaskPluginManager { } - public Map getTaskChannelMap() { + public static Map getTaskChannelMap() { return Collections.unmodifiableMap(taskChannelMap); } - public Map getTaskChannelFactoryMap() { + public static Map getTaskChannelFactoryMap() { return Collections.unmodifiableMap(taskChannelFactoryMap); } - public TaskChannel getTaskChannel(String type) { - return this.getTaskChannelMap().get(type); + public static TaskChannel getTaskChannel(String type) { + return getTaskChannelMap().get(type); } - public boolean checkTaskParameters(ParametersNode parametersNode) { - AbstractParameters abstractParameters = this.getParameters(parametersNode); + public static boolean checkTaskParameters(ParametersNode parametersNode) { + AbstractParameters abstractParameters = getParameters(parametersNode); return abstractParameters != null && abstractParameters.checkParameters(); } - public AbstractParameters getParameters(ParametersNode parametersNode) { + public static AbstractParameters getParameters(ParametersNode parametersNode) { String taskType = parametersNode.getTaskType(); if (Objects.isNull(taskType)) { return null; @@ -106,7 +103,7 @@ public class TaskPluginManager { case TaskConstants.TASK_TYPE_DYNAMIC: return JSONUtils.parseObject(parametersNode.getTaskParams(), DynamicParameters.class); default: - TaskChannel taskChannel = this.getTaskChannelMap().get(taskType); + TaskChannel taskChannel = getTaskChannelMap().get(taskType); if (Objects.isNull(taskChannel)) { return null; } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java index aa9553a308..0588b20dda 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java @@ -23,12 +23,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.ImportAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; -@ImportAutoConfiguration(DaoConfiguration.class) +@Import(DaoConfiguration.class) @SpringBootApplication public class UpgradeDolphinScheduler { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index b0866f7fd8..8dd842a8ed 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -24,11 +25,12 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -47,24 +49,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; -@SpringBootApplication -@EnableTransactionManagement -@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = JdbcRegistryAutoConfiguration.class) -}) @Slf4j +@Import({CommonConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) +@SpringBootApplication public class WorkerServer implements IStoppable { @Autowired private WorkerRegistryClient workerRegistryClient; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private WorkerRpcServer workerRpcServer; @@ -89,7 +85,7 @@ public class WorkerServer implements IStoppable { @PostConstruct public void run() { this.workerRpcServer.start(); - this.taskPluginManager.loadPlugin(); + TaskPluginManager.loadPlugin(); this.workerRegistryClient.setRegistryStoppable(this); this.workerRegistryClient.start(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java index 19421ee05e..ea44f1790f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -35,13 +34,11 @@ public class DefaultWorkerTaskExecutor extends WorkerTaskExecutor { public DefaultWorkerTaskExecutor(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java index 0141a5cd17..20fa6a5e2a 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -35,20 +34,17 @@ public class DefaultWorkerTaskExecutorFactory private final @NonNull TaskExecutionContext taskExecutionContext; private final @NonNull WorkerConfig workerConfig; private final @NonNull WorkerMessageSender workerMessageSender; - private final @NonNull TaskPluginManager taskPluginManager; private final @Nullable StorageOperate storageOperate; private final @NonNull WorkerRegistryClient workerRegistryClient; public DefaultWorkerTaskExecutorFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; } @@ -59,7 +55,6 @@ public class DefaultWorkerTaskExecutorFactory taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java index f713605a48..41cef62028 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java @@ -75,7 +75,6 @@ public abstract class WorkerTaskExecutor implements Runnable { protected final TaskExecutionContext taskExecutionContext; protected final WorkerConfig workerConfig; protected final WorkerMessageSender workerMessageSender; - protected final TaskPluginManager taskPluginManager; protected final @Nullable StorageOperate storageOperate; protected final WorkerRegistryClient workerRegistryClient; @@ -85,13 +84,11 @@ public abstract class WorkerTaskExecutor implements Runnable { @NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); @@ -220,7 +217,7 @@ public abstract class WorkerTaskExecutor implements Runnable { log.info("WorkflowInstanceExecDir: {} check successfully", taskExecutionContext.getExecutePath()); TaskChannel taskChannel = - Optional.ofNullable(taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) + Optional.ofNullable(TaskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) .orElseThrow(() -> new TaskPluginException(taskExecutionContext.getTaskType() + " task plugin not found, please check the task type is correct.")); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java index 599746818d..56f3207884 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -36,12 +35,6 @@ public class WorkerTaskExecutorFactoryBuilder { @Autowired private WorkerMessageSender workerMessageSender; - @Autowired - private TaskPluginManager taskPluginManager; - - @Autowired - private WorkerTaskExecutorThreadPool workerManager; - @Autowired(required = false) private StorageOperate storageOperate; @@ -51,14 +44,11 @@ public class WorkerTaskExecutorFactoryBuilder { public WorkerTaskExecutorFactoryBuilder( WorkerConfig workerConfig, WorkerMessageSender workerMessageSender, - TaskPluginManager taskPluginManager, WorkerTaskExecutorThreadPool workerManager, StorageOperate storageOperate, WorkerRegistryClient workerRegistryClient) { this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.taskPluginManager = taskPluginManager; - this.workerManager = workerManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; } @@ -67,7 +57,6 @@ public class WorkerTaskExecutorFactoryBuilder { return new DefaultWorkerTaskExecutorFactory(taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java index 43ef6f87d0..e211fcdd1f 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -40,8 +39,6 @@ public class DefaultWorkerTaskExecutorTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); - private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); private WorkerRegistryClient workerRegistryClient = Mockito.mock(WorkerRegistryClient.class); @@ -58,7 +55,6 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); @@ -82,7 +78,6 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java index 988f1f7fec..182ac6a1c2 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.TaskExecuteThreadsFullPolicy; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -68,7 +67,7 @@ class WorkerTaskExecutorThreadPoolTest { protected MockWorkerTaskExecutor(Runnable runnable) { super(TaskExecutionContext.builder().taskInstanceId((int) System.nanoTime()).build(), new WorkerConfig(), - new WorkerMessageSender(), new TaskPluginManager(), new StorageOperate() { + new WorkerMessageSender(), new StorageOperate() { @Override public void createTenantDirIfNotExists(String tenantCode) { diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java index 592340214f..f761dd61d6 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java @@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHost import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; @@ -70,8 +69,6 @@ public class TaskInstanceOperationFunctionTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); - private WorkerTaskExecutorThreadPool workerManager = Mockito.mock(WorkerTaskExecutorThreadPool.class); private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); @@ -94,7 +91,6 @@ public class TaskInstanceOperationFunctionTest { WorkerTaskExecutorFactoryBuilder workerTaskExecutorFactoryBuilder = new WorkerTaskExecutorFactoryBuilder( workerConfig, workerMessageSender, - taskPluginManager, workerManager, storageOperate, workerRegistryClient); @@ -189,7 +185,6 @@ public class TaskInstanceOperationFunctionTest { WorkerTaskExecutorFactoryBuilder workerTaskExecutorFactoryBuilder = new WorkerTaskExecutorFactoryBuilder( workerConfig, workerMessageSender, - taskPluginManager, workerManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/test/resources/logback.xml b/dolphinscheduler-worker/src/test/resources/logback.xml index d63ea4a0b5..10fc24d974 100644 --- a/dolphinscheduler-worker/src/test/resources/logback.xml +++ b/dolphinscheduler-worker/src/test/resources/logback.xml @@ -22,7 +22,8 @@ - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - + [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n UTF-8 @@ -60,18 +61,15 @@ - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - + [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n UTF-8 - - - - - - + + From e66441a2c9ec38387b34f3712055af7308876efc Mon Sep 17 00:00:00 2001 From: privking <43061765+privking@users.noreply.github.com> Date: Sat, 20 Apr 2024 07:32:18 +0800 Subject: [PATCH 65/88] [FIX] Fix cannot recover a stopped workflow instance (#15880) --- .../workflow/instance/pause/recover/RecoverExecuteFunction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java index 149e1abd29..34bd2561b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java @@ -43,7 +43,7 @@ public class RecoverExecuteFunction implements ExecuteFunction Date: Mon, 22 Apr 2024 15:05:59 +0800 Subject: [PATCH 66/88] [Improvement][Spark] Support Local Spark Cluster (#15589) * [Improvement][Spark] Support Local Spark Cluster * remote default local from deploy mode --------- Co-authored-by: Rick Cheng --- docs/docs/en/guide/task/spark.md | 1 + docs/docs/zh/guide/task/spark.md | 1 + .../plugin/task/spark/SparkParameters.java | 5 + .../plugin/task/spark/SparkTask.java | 23 ++-- .../task/spark/SparkParametersTest.java | 1 - .../plugin/task/spark/SparkTaskTest.java | 109 +++++++++++++++--- .../src/locales/en_US/project.ts | 2 + .../src/locales/zh_CN/project.ts | 2 + .../task/components/node/fields/use-spark.ts | 26 +++++ .../task/components/node/format-data.ts | 1 + .../task/components/node/tasks/use-spark.ts | 3 +- 11 files changed, 151 insertions(+), 23 deletions(-) diff --git a/docs/docs/en/guide/task/spark.md b/docs/docs/en/guide/task/spark.md index 3e0f83b253..930f2cd0b0 100644 --- a/docs/docs/en/guide/task/spark.md +++ b/docs/docs/en/guide/task/spark.md @@ -24,6 +24,7 @@ Spark task type for executing Spark application. When executing the Spark task, |----------------------------|------------------------------------------------------------------------------------------------------------------------------------| | Program type | Supports Java, Scala, Python, and SQL. | | The class of main function | The **full path** of Main Class, the entry point of the Spark program. | +| Master | The The master URL for the cluster. | | Main jar package | The Spark jar package (upload by Resource Center). | | SQL scripts | SQL statements in .sql files that Spark sql runs. | | Deployment mode |

  • spark submit supports three modes: cluster, client and local.
  • spark sql supports client and local modes.
| diff --git a/docs/docs/zh/guide/task/spark.md b/docs/docs/zh/guide/task/spark.md index a392f55826..2f7b2ee346 100644 --- a/docs/docs/zh/guide/task/spark.md +++ b/docs/docs/zh/guide/task/spark.md @@ -23,6 +23,7 @@ Spark 任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支 - 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。 - 主函数的 Class：Spark 程序的入口 Main class 的全路径。 - 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。 +- Master：执行 Spark 集群的 Master Url。 - SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。 - 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。 (2) spark sql 支持 client 和 local 两种模式。 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java index c5fcb5b76b..873ba22c71 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java @@ -38,6 +38,11 @@ public class SparkParameters extends AbstractParameters { */ private String mainClass; + /** + * master url + */ + private String master; + /** * deploy mode local / cluster / client */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index a0d1f3fc77..3c5fc17698 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -124,22 +124,31 @@ public class SparkTask extends AbstractYarnTask { */ private List populateSparkOptions() { List args = new ArrayList<>(); - args.add(SparkConstants.MASTER); + // see https://spark.apache.org/docs/latest/submitting-applications.html + // TODO remove the option 'local' from deploy-mode String deployMode = StringUtils.isNotEmpty(sparkParameters.getDeployMode()) ? sparkParameters.getDeployMode() : SparkConstants.DEPLOY_MODE_LOCAL; + boolean onLocal = SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode); boolean onNativeKubernetes = StringUtils.isNotEmpty(sparkParameters.getNamespace()); - String masterUrl = onNativeKubernetes ? SPARK_ON_K8S_MASTER_PREFIX + - Config.fromKubeconfig(taskExecutionContext.getK8sTaskExecutionContext().getConfigYaml()).getMasterUrl() - : SparkConstants.SPARK_ON_YARN; + String masterUrl = StringUtils.isNotEmpty(sparkParameters.getMaster()) ? sparkParameters.getMaster() + : onLocal ? deployMode + : onNativeKubernetes + ? SPARK_ON_K8S_MASTER_PREFIX + Config + .fromKubeconfig( + taskExecutionContext.getK8sTaskExecutionContext().getConfigYaml()) + .getMasterUrl() + : SparkConstants.SPARK_ON_YARN; - if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode)) { - args.add(masterUrl); + args.add(SparkConstants.MASTER); + args.add(masterUrl); + + if (!onLocal) { args.add(SparkConstants.DEPLOY_MODE); + args.add(deployMode); } - args.add(deployMode); ProgramType programType = sparkParameters.getProgramType(); String mainClass = sparkParameters.getMainClass(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java index ab164f2eb5..19ec707c62 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java @@ -54,6 +54,5 @@ public class SparkParametersTest { resourceFilesList = sparkParameters.getResourceFilesList(); Assertions.assertNotNull(resourceFilesList); Assertions.assertEquals(3, resourceFilesList.size()); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index 78d8968e59..5138562564 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -17,16 +17,27 @@ package org.apache.dolphinscheduler.plugin.task.spark; +import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.TYPE_FILE; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -41,20 +52,33 @@ public class SparkTaskTest { Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("5536"); - SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); - sparkTask.init(); - Assertions.assertEquals( - "${SPARK_HOME}/bin/spark-sql " + - "--master yarn " + - "--deploy-mode client " + - "--conf spark.driver.cores=1 " + - "--conf spark.driver.memory=512M " + - "--conf spark.executor.instances=2 " + - "--conf spark.executor.cores=2 " + - "--conf spark.executor.memory=1G " + - "--name sparksql " + - "-f /tmp/5536_node.sql", - sparkTask.getScript()); + ResourceContext resourceContext = Mockito.mock(ResourceContext.class); + Mockito.when(taskExecutionContext.getResourceContext()).thenReturn(resourceContext); + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInLocal("test"); + Mockito.when(resourceContext.getResourceItem(any())).thenReturn(resourceItem); + + try (MockedStatic fileUtilsMockedStatic = Mockito.mockStatic(FileUtils.class)) { + fileUtilsMockedStatic + .when(() -> FileUtils + .readFileToString(any(File.class), any(Charset.class))) + .thenReturn("test"); + + SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); + sparkTask.init(); + Assertions.assertEquals( + "${SPARK_HOME}/bin/spark-sql " + + "--master yarn " + + "--deploy-mode client " + + "--conf spark.driver.cores=1 " + + "--conf spark.driver.memory=512M " + + "--conf spark.executor.instances=2 " + + "--conf spark.executor.cores=2 " + + "--conf spark.executor.memory=1G " + + "--name sparksql " + + "-f /tmp/5536_node.sql", + sparkTask.getScript()); + } } @Test @@ -86,11 +110,41 @@ public class SparkTaskTest { sparkTask.getScript()); } + @Test + public void testBuildCommandWithSparkSubmitMaster() { + String parameters = buildSparkParametersWithMaster(); + TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInStorage("/lib/dolphinscheduler-task-spark.jar"); + resourceItem.setResourceAbsolutePathInLocal("/lib/dolphinscheduler-task-spark.jar"); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem); + + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters); + Mockito.when(taskExecutionContext.getResourceContext()).thenReturn(resourceContext); + SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); + sparkTask.init(); + Assertions.assertEquals( + "${SPARK_HOME}/bin/spark-submit " + + "--master spark://localhost:7077 " + + "--deploy-mode client " + + "--class org.apache.dolphinscheduler.plugin.task.spark.SparkTaskTest " + + "--conf spark.driver.cores=1 " + + "--conf spark.driver.memory=512M " + + "--conf spark.executor.instances=2 " + + "--conf spark.executor.cores=2 " + + "--conf spark.executor.memory=1G " + + "--name spark " + + "/lib/dolphinscheduler-task-spark.jar", + sparkTask.getScript()); + } + private String buildSparkParametersWithSparkSql() { SparkParameters sparkParameters = new SparkParameters(); sparkParameters.setLocalParams(Collections.emptyList()); sparkParameters.setRawScript("selcet 11111;"); sparkParameters.setProgramType(ProgramType.SQL); + sparkParameters.setSqlExecutionType(TYPE_FILE); sparkParameters.setMainClass(""); sparkParameters.setDeployMode("client"); sparkParameters.setAppName("sparksql"); @@ -100,6 +154,13 @@ public class SparkTaskTest { sparkParameters.setNumExecutors(2); sparkParameters.setExecutorMemory("1G"); sparkParameters.setExecutorCores(2); + + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setResourceName("testSparkParameters1.jar"); + List resourceInfos = new ArrayList<>(Arrays.asList( + resourceInfo1)); + sparkParameters.setResourceList(resourceInfos); + return JSONUtils.toJsonString(sparkParameters); } @@ -122,4 +183,24 @@ public class SparkTaskTest { return JSONUtils.toJsonString(sparkParameters); } + private String buildSparkParametersWithMaster() { + SparkParameters sparkParameters = new SparkParameters(); + sparkParameters.setLocalParams(Collections.emptyList()); + sparkParameters.setProgramType(ProgramType.SCALA); + sparkParameters.setMainClass("org.apache.dolphinscheduler.plugin.task.spark.SparkTaskTest"); + sparkParameters.setDeployMode("client"); + sparkParameters.setAppName("spark"); + sparkParameters.setMaster("spark://localhost:7077"); + sparkParameters.setOthers(""); + sparkParameters.setDriverCores(1); + sparkParameters.setDriverMemory("512M"); + sparkParameters.setNumExecutors(2); + sparkParameters.setExecutorMemory("1G"); + sparkParameters.setExecutorCores(2); + ResourceInfo resourceInfo = new ResourceInfo(); + resourceInfo.setResourceName("/lib/dolphinscheduler-task-spark.jar"); + sparkParameters.setMainJar(resourceInfo); + return JSONUtils.toJsonString(sparkParameters); + } + } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index cb50b19fc7..7a39752526 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -447,6 +447,8 @@ export default { timeout_period_tips: 'Timeout must be a positive integer', script: 'Script', script_tips: 'Please enter script(required)', + master: 'Master', + master_tips: 'Please enter master url(required)', init_script: 'Initialization script', init_script_tips: 'Please enter initialization script', resources: 'Resources', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 6865a49abc..50e0a821ef 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -437,6 +437,8 @@ export default { timeout_period_tips: '超时时长必须为正整数', script: '脚本', script_tips: '请输入脚本（必填）', + master: 'Master', + master_tips: '请输入master url（必填）', init_script: '初始化脚本', init_script_tips: '请输入初始化脚本', resources: '资源', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts index ad7fb77fa9..ab89e69e6d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts @@ -37,6 +37,10 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { model.programType === 'PYTHON' || model.programType === 'SQL' ? 0 : 24 ) + const masterSpan = computed(() => + model.programType === 'PYTHON' || model.programType === 'SQL' ? 0 : 24 + ) + const mainArgsSpan = computed(() => (model.programType === 'SQL' ? 0 : 24)) const rawScriptSpan = computed(() => @@ -138,6 +142,28 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { message: t('project.node.script_tips') } }, + { + type: 'input', + field: 'master', + span: masterSpan, + name: t('project.node.master'), + props: { + placeholder: t('project.node.master_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: false, + validator(validate: any, value: string) { + if ( + model.programType !== 'PYTHON' && + !value && + model.programType !== 'SQL' + ) { + return new Error(t('project.node.master_tips')) + } + } + } + }, useDeployMode(24, ref(true), showCluster), useNamespace(), { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index ef9e5dec61..2ab1712b6f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -68,6 +68,7 @@ export function formatParams(data: INodeData): { } if (data.taskType === 'SPARK') { + taskParams.master = data.master taskParams.driverCores = data.driverCores taskParams.driverMemory = data.driverMemory taskParams.numExecutors = data.numExecutors diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts index 05aa0fe1c6..1e5c929b0f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-spark.ts @@ -45,7 +45,8 @@ export function useSpark({ timeout: 30, programType: 'SCALA', rawScript: '', - deployMode: 'local', + master: '', + deployMode: '', driverCores: 1, driverMemory: '512M', numExecutors: 2, From e2c8b080f90c71aa6045609a84ffde9272892fea Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 22 Apr 2024 17:05:24 +0800 Subject: [PATCH 67/88] [DSIP-31] Reduce the thread pool size of hikari (#15890) --- .../src/main/resources/application.yaml | 8 -------- dolphinscheduler-api/src/main/resources/application.yaml | 8 -------- .../src/main/resources/application.yaml | 8 -------- .../src/main/resources/application.yaml | 8 -------- .../src/test/resources/application-mysql.yaml | 8 -------- .../src/test/resources/application-postgresql.yaml | 8 -------- 6 files changed, 48 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 0b28d6c8b0..0dbb6988ce 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -30,15 +30,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 # Mybatis-plus configuration, you don't need to change it mybatis-plus: diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 61f9c8a592..e38d0c5a8e 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -51,15 +51,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 quartz: auto-startup: false job-store-type: jdbc diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index c2d8f5e787..f18c6ef61d 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -29,15 +29,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 quartz: job-store-type: jdbc jdbc: diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml index 136a4c5fd4..23680728dc 100644 --- a/dolphinscheduler-tools/src/main/resources/application.yaml +++ b/dolphinscheduler-tools/src/main/resources/application.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 # Mybatis-plus configuration, you don't need to change it mybatis-plus: diff --git a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml index 0b553c2bda..d33a68a7f8 100644 --- a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 mybatis-plus: mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml diff --git a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml index 21daaa0606..a8be29014f 100644 --- a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml @@ -24,15 +24,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 mybatis-plus: mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml From e9d85914d78197314bee585500d94a6a90733789 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 22 Apr 2024 17:41:17 +0800 Subject: [PATCH 68/88] Fix queryByTypeAndJobId might error due to multiple result (#15883) --- .../dao/mapper/TriggerRelationMapper.java | 2 +- .../dao/mapper/TriggerRelationMapperTest.java | 26 ++++----- dolphinscheduler-dist/release-docs/LICENSE | 2 +- .../process/TriggerRelationService.java | 4 +- .../process/TriggerRelationServiceImpl.java | 41 ++++++++++---- .../process/TriggerRelationServiceTest.java | 56 ++++++++----------- pom.xml | 7 +++ tools/dependencies/known-dependencies.txt | 4 +- 8 files changed, 79 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java index 10a0acf47f..912ef28101 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java @@ -36,7 +36,7 @@ public interface TriggerRelationMapper extends BaseMapper { * @param jobId * @return */ - TriggerRelation queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); + List queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); /** * query triggerRelation by code diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java index d3f4fcc666..7e31b64a23 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapperTest.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; -import java.util.List; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -67,9 +67,9 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTypeAndJobId() { TriggerRelation expectRelation = createTriggerRelation(); - TriggerRelation actualRelation = triggerRelationMapper.queryByTypeAndJobId( - expectRelation.getTriggerType(), expectRelation.getJobId()); - Assertions.assertEquals(expectRelation, actualRelation); + assertThat( + triggerRelationMapper.queryByTypeAndJobId(expectRelation.getTriggerType(), expectRelation.getJobId())) + .containsExactly(expectRelation); } /** @@ -80,9 +80,8 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTriggerRelationCode() { TriggerRelation expectRelation = createTriggerRelation(); - List actualRelations = triggerRelationMapper.queryByTriggerRelationCode( - expectRelation.getTriggerCode()); - Assertions.assertEquals(actualRelations.size(), 1); + assertThat(triggerRelationMapper.queryByTriggerRelationCode(expectRelation.getTriggerCode())) + .containsExactly(expectRelation); } /** @@ -93,17 +92,15 @@ public class TriggerRelationMapperTest extends BaseDaoTest { @Test public void testQueryByTriggerRelationCodeAndType() { TriggerRelation expectRelation = createTriggerRelation(); - List actualRelations = triggerRelationMapper.queryByTriggerRelationCodeAndType( - expectRelation.getTriggerCode(), expectRelation.getTriggerType()); - Assertions.assertEquals(actualRelations.size(), 1); + assertThat(triggerRelationMapper.queryByTriggerRelationCodeAndType(expectRelation.getTriggerCode(), + expectRelation.getTriggerType())).containsExactly(expectRelation); } @Test public void testUpsert() { TriggerRelation expectRelation = createTriggerRelation(); triggerRelationMapper.upsert(expectRelation); - TriggerRelation actualRelation = triggerRelationMapper.selectById(expectRelation.getId()); - Assertions.assertEquals(expectRelation, actualRelation); + assertThat(triggerRelationMapper.selectById(expectRelation.getId())).isEqualTo(expectRelation); } /** @@ -113,8 +110,7 @@ public class TriggerRelationMapperTest extends BaseDaoTest { public void testDelete() { TriggerRelation expectRelation = createTriggerRelation(); triggerRelationMapper.deleteById(expectRelation.getId()); - TriggerRelation actualRelation = triggerRelationMapper.selectById(expectRelation.getId()); - Assertions.assertNull(actualRelation); + assertThat(triggerRelationMapper.selectById(expectRelation.getId())).isNull(); } /** diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 8856138aa4..84832b2b32 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -528,7 +528,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. nimbus-jose-jwt 9.22: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.22, Apache 2.0 woodstox-core 6.4.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/6.4.0, Apache 2.0 auto-value 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value/1.10.1, Apache 2.0 - auto-value-annotations 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.1, Apache 2.0 + auto-value-annotations 1.10.4: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.4, Apache 2.0 conscrypt-openjdk-uber 2.5.2: https://mvnrepository.com/artifact/org.conscrypt/conscrypt-openjdk-uber/2.5.2, Apache 2.0 gapic-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/gapic-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0 google-api-client 2.2.0: https://mvnrepository.com/artifact/com.google.api-client/google-api-client/2.2.0, Apache 2.0 diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java index b78f477931..6de2506afd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationService.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.service.process; import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; +import java.util.List; + import org.springframework.stereotype.Component; /** @@ -30,7 +32,7 @@ public interface TriggerRelationService { void saveTriggerToDb(ApiTriggerType type, Long triggerCode, Integer jobId); - TriggerRelation queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId); + List queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId); int saveCommandTrigger(Integer commandId, Integer processInstanceId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java index df41c41eef..0344ea87ea 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceImpl.java @@ -21,14 +21,20 @@ import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; import org.apache.dolphinscheduler.dao.mapper.TriggerRelationMapper; +import org.apache.commons.collections4.CollectionUtils; + import java.util.Date; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** - * Trigger relation operator to db + * Trigger relation operator to db */ +@Slf4j @Component public class TriggerRelationServiceImpl implements TriggerRelationService { @@ -45,29 +51,44 @@ public class TriggerRelationServiceImpl implements TriggerRelationService { triggerRelation.setUpdateTime(new Date()); triggerRelationMapper.upsert(triggerRelation); } + @Override - public TriggerRelation queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId) { + public List queryByTypeAndJobId(ApiTriggerType apiTriggerType, int jobId) { return triggerRelationMapper.queryByTypeAndJobId(apiTriggerType.getCode(), jobId); } @Override public int saveCommandTrigger(Integer commandId, Integer processInstanceId) { - TriggerRelation exist = queryByTypeAndJobId(ApiTriggerType.PROCESS, processInstanceId); - if (exist == null) { + List existTriggers = queryByTypeAndJobId(ApiTriggerType.PROCESS, processInstanceId); + if (CollectionUtils.isEmpty(existTriggers)) { return 0; } - saveTriggerToDb(ApiTriggerType.COMMAND, exist.getTriggerCode(), commandId); - return 1; + existTriggers.forEach(triggerRelation -> saveTriggerToDb(ApiTriggerType.COMMAND, + triggerRelation.getTriggerCode(), commandId)); + int triggerRelationSize = existTriggers.size(); + if (triggerRelationSize > 1) { + // Fix https://github.com/apache/dolphinscheduler/issues/15864 + // This case shouldn't happen + log.error("The PROCESS TriggerRelation of command: {} is more than one", commandId); + } + return existTriggers.size(); } @Override public int saveProcessInstanceTrigger(Integer commandId, Integer processInstanceId) { - TriggerRelation exist = queryByTypeAndJobId(ApiTriggerType.COMMAND, commandId); - if (exist == null) { + List existTriggers = queryByTypeAndJobId(ApiTriggerType.COMMAND, commandId); + if (CollectionUtils.isEmpty(existTriggers)) { return 0; } - saveTriggerToDb(ApiTriggerType.PROCESS, exist.getTriggerCode(), processInstanceId); - return 1; + existTriggers.forEach(triggerRelation -> saveTriggerToDb(ApiTriggerType.PROCESS, + triggerRelation.getTriggerCode(), processInstanceId)); + int triggerRelationSize = existTriggers.size(); + if (triggerRelationSize > 1) { + // Fix https://github.com/apache/dolphinscheduler/issues/15864 + // This case shouldn't happen + log.error("The COMMAND TriggerRelation of command: {} is more than one", commandId); + } + return existTriggers.size(); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java index 8f4790111c..4f3bbb0bc7 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/TriggerRelationServiceTest.java @@ -17,24 +17,26 @@ package org.apache.dolphinscheduler.service.process; +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.common.enums.ApiTriggerType; import org.apache.dolphinscheduler.dao.entity.TriggerRelation; import org.apache.dolphinscheduler.dao.mapper.TriggerRelationMapper; -import org.apache.dolphinscheduler.service.cron.CronUtilsTest; import java.util.Date; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * Trigger Relation Service Test @@ -43,8 +45,6 @@ import org.slf4j.LoggerFactory; @MockitoSettings(strictness = Strictness.LENIENT) public class TriggerRelationServiceTest { - private static final Logger logger = LoggerFactory.getLogger(CronUtilsTest.class); - @InjectMocks private TriggerRelationServiceImpl triggerRelationService; @Mock @@ -52,47 +52,37 @@ public class TriggerRelationServiceTest { @Test public void saveTriggerToDb() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); + doNothing().when(triggerRelationMapper).upsert(any()); triggerRelationService.saveTriggerToDb(ApiTriggerType.COMMAND, 1234567890L, 100); } @Test public void queryByTypeAndJobId() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); - TriggerRelation triggerRelation1 = triggerRelationService.queryByTypeAndJobId( - ApiTriggerType.PROCESS, 100); - Assertions.assertNotNull(triggerRelation1); - TriggerRelation triggerRelation2 = triggerRelationService.queryByTypeAndJobId( - ApiTriggerType.PROCESS, 200); - Assertions.assertNull(triggerRelation2); + assertThat(triggerRelationService.queryByTypeAndJobId(ApiTriggerType.PROCESS, 100)).hasSize(1); + assertThat(triggerRelationService.queryByTypeAndJobId(ApiTriggerType.PROCESS, 200)).isEmpty(); } @Test public void saveCommandTrigger() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); - int result = -1; - result = triggerRelationService.saveCommandTrigger(1234567890, 100); - Assertions.assertTrue(result > 0); - result = triggerRelationService.saveCommandTrigger(1234567890, 200); - Assertions.assertTrue(result == 0); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.PROCESS.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); + assertThat(triggerRelationService.saveCommandTrigger(1234567890, 100)).isAtLeast(1); + assertThat(triggerRelationService.saveCommandTrigger(1234567890, 200)).isEqualTo(0); } @Test public void saveProcessInstanceTrigger() { - Mockito.doNothing().when(triggerRelationMapper).upsert(Mockito.any()); - Mockito.when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.COMMAND.getCode(), 100)) - .thenReturn(getTriggerTdoDb()); - int result = -1; - result = triggerRelationService.saveProcessInstanceTrigger(100, 1234567890); - Assertions.assertTrue(result > 0); - result = triggerRelationService.saveProcessInstanceTrigger(200, 1234567890); - Assertions.assertTrue(result == 0); + doNothing().when(triggerRelationMapper).upsert(any()); + when(triggerRelationMapper.queryByTypeAndJobId(ApiTriggerType.COMMAND.getCode(), 100)) + .thenReturn(Lists.newArrayList(getTriggerTdoDb())); + assertThat(triggerRelationService.saveProcessInstanceTrigger(100, 1234567890)).isAtLeast(1); + assertThat(triggerRelationService.saveProcessInstanceTrigger(200, 1234567890)).isEqualTo(0); } private TriggerRelation getTriggerTdoDb() { diff --git a/pom.xml b/pom.xml index 4a71741f8e..dd4cb1adf1 100755 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 7.1.2 1.18.20 4.2.0 + 1.4.2 apache ${project.name} ${project.version} @@ -372,6 +373,12 @@ ${awaitility.version} test + + com.google.truth + truth + ${truth.version} + test + diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 6c2e89fb37..33987119db 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -9,7 +9,7 @@ animal-sniffer-annotations-1.19.jar annotations-2.17.282.jar annotations-13.0.jar apache-client-2.17.282.jar -asm-9.1.jar +asm-9.6.jar aspectjweaver-1.9.7.jar aspectjrt-1.9.7.jar auth-2.17.282.jar @@ -434,7 +434,7 @@ woodstox-core-6.4.0.jar azure-core-management-1.10.1.jar api-common-2.6.0.jar auto-value-1.10.1.jar -auto-value-annotations-1.10.1.jar +auto-value-annotations-1.10.4.jar bcpkix-jdk15on-1.67.jar bcprov-jdk15on-1.67.jar conscrypt-openjdk-uber-2.5.2.jar From 59f060e278ad0c2400e4dea6359ffda8b1f71d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Tue, 23 Apr 2024 15:10:08 +0800 Subject: [PATCH 69/88] [Improvement] Fix alert code smell --- .../plugin/alert/dingtalk/DingTalkSender.java | 28 ++++++------------- .../alert/dingtalk/DingTalkSenderTest.java | 3 +- .../plugin/alert/email/EmailConstants.java | 2 -- .../plugin/alert/email/MailSender.java | 6 ++-- .../plugin/alert/email/ExcelUtilsTest.java | 11 ++++++-- .../plugin/alert/email/MailUtilsTest.java | 22 +++++++-------- .../plugin/alert/feishu/FeiShuSender.java | 3 +- .../plugin/alert/http/HttpSender.java | 9 +++--- .../prometheus/PrometheusAlertSender.java | 3 +- .../plugin/alert/script/ProcessUtilsTest.java | 3 +- .../plugin/alert/slack/SlackSender.java | 5 ++-- .../plugin/alert/telegram/TelegramSender.java | 2 +- .../alert/wechat/WeChatAlertConstants.java | 2 -- .../plugin/alert/wechat/WeChatSender.java | 27 ++++++------------ .../common/constants/Constants.java | 5 ---- .../common/utils/FileUtils.java | 3 +- .../common/utils/HttpUtils.java | 3 +- 17 files changed, 60 insertions(+), 77 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index c0070ac11c..c8ded8cfad 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -48,6 +48,8 @@ import java.util.Objects; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -189,7 +191,7 @@ public final class DingTalkSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -317,15 +319,17 @@ public final class DingTalkSender { String sign = org.apache.commons.lang3.StringUtils.EMPTY; try { Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); - byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); - sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + byte[] signData = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); + sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error("generate sign error, message:{}", e); } return url + "×tamp=" + timestamp + "&sign=" + sign; } + @Getter + @Setter static final class DingTalkSendMsgResponse { private Integer errcode; @@ -334,22 +338,6 @@ public final class DingTalkSender { public DingTalkSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - @Override public boolean equals(final Object o) { if (o == this) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index 7a8df0549c..cd30105c7a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; import org.apache.dolphinscheduler.alert.api.AlertResult; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -47,7 +48,7 @@ public class DingTalkSenderTest { @Test public void testSend() { DingTalkSender dingTalkSender = new DingTalkSender(dingTalkConfig); - dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8"); + dingTalkSender.sendDingTalkMsg("keyWord+Welcome", StandardCharsets.UTF_8.name()); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkSender = new DingTalkSender(dingTalkConfig); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test"); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java index 94eb4efa39..3eec7022fd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java @@ -56,8 +56,6 @@ public final class EmailConstants { public static final String TABLE_BODY_HTML_TAIL = ""; - public static final String UTF_8 = "UTF-8"; - public static final String EXCEL_SUFFIX_XLSX = ".xlsx"; public static final String SINGLE_SLASH = "/"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index 2e400efbce..8826a44fba 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -34,6 +34,7 @@ import org.apache.commons.mail.HtmlEmail; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -171,7 +172,7 @@ public final class MailSender { Session session = getSession(); email.setMailSession(session); email.setFrom(mailSenderEmail); - email.setCharset(EmailConstants.UTF_8); + email.setCharset(StandardCharsets.UTF_8.name()); if (CollectionUtils.isNotEmpty(receivers)) { // receivers mail for (String receiver : receivers) { @@ -344,7 +345,8 @@ public final class MailSender { ExcelUtils.genExcelFile(content, randomFilename, xlsFilePath); part2.attachFile(file); - part2.setFileName(MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, EmailConstants.UTF_8, "B")); + part2.setFileName( + MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, StandardCharsets.UTF_8.name(), "B")); // add components to collection partList.addBodyPart(part1); partList.addBodyPart(part2); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java index f428a16d92..f28df77de9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java @@ -66,8 +66,15 @@ public class ExcelUtilsTest { @Test public void testGenExcelFileByCheckDir() { - ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls"); - File file = new File("/tmp/xls" + EmailConstants.SINGLE_SLASH + "t" + EmailConstants.EXCEL_SUFFIX_XLSX); + String path = "/tmp/xls"; + ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", path); + File file = + new File( + path + + EmailConstants.SINGLE_SLASH + + "t" + + EmailConstants.EXCEL_SUFFIX_XLSX); file.delete(); + Assertions.assertFalse(file.exists()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index f562999be0..acc255ae0e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -18,10 +18,9 @@ package org.apache.dolphinscheduler.plugin.alert.email; import org.apache.dolphinscheduler.alert.api.AlertConstants; +import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.api.ShowType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate; -import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate; import java.util.ArrayList; import java.util.HashMap; @@ -42,7 +41,6 @@ public class MailUtilsTest { private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class); static MailSender mailSender; private static Map emailConfig = new HashMap<>(); - private static AlertTemplate alertTemplate; @BeforeAll public static void initEmailConfig() { @@ -59,7 +57,6 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, "347801120@qq.com"); emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, "347801120@qq.com"); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp()); - alertTemplate = new DefaultHTMLTemplate(); mailSender = new MailSender(emailConfig); } @@ -77,9 +74,10 @@ public class MailUtilsTest { + "\"Host: 192.168.xx.xx\"," + "\"Notify group :4\"]"; - mailSender.sendMails( + AlertResult alertResult = mailSender.sendMails( "Mysql Exception", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -108,7 +106,8 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "user"); emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd"); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } public String list2String() { @@ -134,7 +133,6 @@ public class MailUtilsTest { logger.info(mapjson); return mapjson; - } @Test @@ -143,7 +141,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -151,7 +150,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -159,7 +159,7 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } - } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index 14a1d63ff0..369060843c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -31,6 +31,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -161,7 +162,7 @@ public final class FeiShuSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java index 999a0c9599..a1de852407 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -58,7 +59,6 @@ public final class HttpSender { * request type get */ private static final String REQUEST_TYPE_GET = "GET"; - private static final String DEFAULT_CHARSET = "utf-8"; private final String headerParams; private final String bodyParams; private final String contentField; @@ -124,7 +124,7 @@ public final class HttpSender { CloseableHttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, DEFAULT_CHARSET); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } private void createHttpRequest(String msg) throws MalformedURLException, URISyntaxException { @@ -157,7 +157,8 @@ public final class HttpSender { type = URL_SPLICE_CHAR; } try { - url = String.format("%s%s%s=%s", url, type, contentField, URLEncoder.encode(msg, DEFAULT_CHARSET)); + url = String.format("%s%s%s=%s", url, type, contentField, + URLEncoder.encode(msg, StandardCharsets.UTF_8.name())); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -190,7 +191,7 @@ public final class HttpSender { } // set msg content field objectNode.put(contentField, msg); - StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), DEFAULT_CHARSET); + StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), StandardCharsets.UTF_8); ((HttpPost) httpRequest).setEntity(entity); } catch (Exception e) { log.error("send http alert msg exception : {}", e.getMessage()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java index d27745049e..1106e6799f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java @@ -34,6 +34,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -90,7 +91,7 @@ public class PrometheusAlertSender { } HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); log.error( "Prometheus alert manager send alert failed, http status code: {}, title: {} ,content: {}, resp: {}", diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java index a34f062264..3d85d5d638 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java @@ -33,6 +33,7 @@ public class ProcessUtilsTest { @Test public void testExecuteScript() { - ProcessUtils.executeScript(cmd); + int code = ProcessUtils.executeScript(cmd); + assert code != -1; } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java index 60b2f8281a..4096ccc998 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java @@ -29,6 +29,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -81,11 +82,11 @@ public final class SlackSender { } HttpPost httpPost = new HttpPost(webHookUrl); - httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), "UTF-8")); + httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, "UTF-8"); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } catch (Exception e) { log.error("Send message to slack error.", e); return "System Exception"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 8aba9f5c2b..129bc62c1c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -153,7 +153,7 @@ public final class TelegramSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java index 7f5eaef4f9..76ad480015 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java @@ -23,8 +23,6 @@ public final class WeChatAlertConstants { static final String MARKDOWN_ENTER = "\n"; - static final String CHARSET = "UTF-8"; - static final String WE_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}"; static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index 4b49e0436d..c5ffec1f46 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -38,6 +38,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -45,6 +46,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -85,12 +88,12 @@ public final class WeChatSender { CloseableHttpClient httpClient = HttpClients.custom().setRetryHandler(HttpServiceRetryStrategy.retryStrategy).build()) { HttpPost httpPost = new HttpPost(url); - httpPost.setEntity(new StringEntity(data, WeChatAlertConstants.CHARSET)); + httpPost.setEntity(new StringEntity(data, StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -142,7 +145,7 @@ public final class WeChatSender { HttpGet httpGet = new HttpGet(url); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } @@ -259,6 +262,8 @@ public final class WeChatSender { return null; } + @Getter + @Setter static final class WeChatSendMsgResponse { private Integer errcode; @@ -267,22 +272,6 @@ public final class WeChatSender { public WeChatSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - public boolean equals(final Object o) { if (o == this) { return true; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index b5bbf740e9..ebf668a312 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -242,11 +242,6 @@ public final class Constants { */ public static final String HTTP_X_REAL_IP = "X-Real-IP"; - /** - * UTF-8 - */ - public static final String UTF_8 = "UTF-8"; - /** * user name regex */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index fee1d9a95c..60629576d9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPA import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE; -import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS; import org.apache.commons.io.IOUtils; @@ -207,7 +206,7 @@ public class FileUtils { while ((length = inputStream.read(buffer)) != -1) { output.write(buffer, 0, length); } - return output.toString(UTF_8); + return output.toString(StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException(e); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 5c79fff951..e5d2256150 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -36,6 +36,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -143,7 +144,7 @@ public class HttpUtils { } HttpEntity entity = response.getEntity(); - return entity != null ? EntityUtils.toString(entity, Constants.UTF_8) : null; + return entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null; } catch (IOException e) { log.error("Error executing HTTP GET request", e); return null; From 1d13ef09c3607870f7309d9f41dc5deac8d209ee Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Wed, 24 Apr 2024 23:31:56 +0800 Subject: [PATCH 70/88] [TEST] increase coverage of project parameter service test (#15905) Co-authored-by: abzymeinsjtu --- .../impl/ProjectParameterServiceImpl.java | 6 +- .../service/ProjectParameterServiceTest.java | 204 +++++++++++++----- 2 files changed, 157 insertions(+), 53 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e0011096e4..17d1d04712 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -227,11 +227,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj } for (ProjectParameter projectParameter : projectParameterList) { - try { - this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); - } catch (Exception e) { - throw new ServiceException(Status.DELETE_PROJECT_PARAMETER_ERROR, e.getMessage()); - } + this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index 7a3fb1b68d..ca51690ce6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -17,27 +17,40 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl; import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectParameter; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; -import org.junit.jupiter.api.Assertions; +import java.util.Collections; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class ProjectParameterServiceTest { @@ -60,92 +73,187 @@ public class ProjectParameterServiceTest { public void testCreateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); + // PERMISSION DENIED + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + + // CODE GENERATION ERROR + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); + } + + // PROJECT_PARAMETER_ALREADY_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // INSERT DATA ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testUpdateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // PROJECT_UPDATE_ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); + assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); ProjectParameter projectParameter = (ProjectParameter) result.getData(); - Assertions.assertNotNull(projectParameter.getOperator()); - Assertions.assertNotNull(projectParameter.getUpdateTime()); + assertNotNull(projectParameter.getOperator()); + assertNotNull(projectParameter.getUpdateTime()); } @Test public void testDeleteProjectParametersByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + + // DATABASE OPERATION ERROR + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(-1); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.DELETE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testQueryProjectParameterByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), - Mockito.any())).thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + Result result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())).thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + @Test + public void testQueryProjectParameterListPaging() { + User loginUser = getGeneralUser(); + Integer pageSize = 10; + Integer pageNo = 1; + + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + + Result result = + projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // SUCCESS + when(projectService.hasProjectAndPerm(any(), any(), any(Result.class), any())) + .thenReturn(true); + + Page page = new Page<>(pageNo, pageSize); + page.setRecords(Collections.singletonList(getProjectParameter())); + + when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page); + result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, + null); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); + } + + @Test + public void testBatchDeleteProjectParametersByCodes() { + User loginUser = getGeneralUser(); + + Result result = projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, ""); + assertEquals(Status.PROJECT_PARAMETER_CODE_EMPTY.getCode(), result.getCode()); + + when(projectParameterMapper.queryByCodes(any())).thenReturn(Collections.singletonList(getProjectParameter())); + + AssertionsHelper.assertThrowsServiceException(Status.PROJECT_PARAMETER_NOT_EXISTS, + () -> projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1,2")); + + projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1"); } private Project getProject(long projectCode) { From b3b8c0784dfc31b516b13601a311bc78550bf931 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 11:05:51 +0800 Subject: [PATCH 71/88] Fix kill dynamic task doesn't kill the wait to run workflow instances (#15896) --- .../dao/mapper/CommandMapper.java | 9 +++- .../dao/mapper/CommandMapper.xml | 7 +++ .../dao/mapper/CommandMapperTest.java | 11 +++- .../runner/task/dynamic/DynamicLogicTask.java | 51 ++++++++++++++++++- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index a8490cbef7..9fb6643227 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -34,8 +34,9 @@ public interface CommandMapper extends BaseMapper { /** * count command state - * @param startTime startTime - * @param endTime endTime + * + * @param startTime startTime + * @param endTime endTime * @param projectCodes projectCodes * @return CommandCount list */ @@ -46,15 +47,19 @@ public interface CommandMapper extends BaseMapper { /** * query command page + * * @return */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); /** * query command page by slot + * * @return command list */ List queryCommandPageBySlot(@Param("limit") int limit, @Param("masterCount") int masterCount, @Param("thisMasterSlot") int thisMasterSlot); + + void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index c950f66413..56db890ef0 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -47,4 +47,11 @@ order by process_instance_priority, id asc limit #{limit} + + delete from t_ds_command + where process_instance_id in + + #{i} + + diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 3d45477d85..2d367e46e4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.dao.mapper; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; @@ -173,6 +175,14 @@ public class CommandMapperTest extends BaseDaoTest { toTestQueryCommandPageBySlot(masterCount, thisMasterSlot); } + @Test + void deleteByWorkflowInstanceIds() { + Command command = createCommand(); + assertThat(commandMapper.selectList(null)).isNotEmpty(); + commandMapper.deleteByWorkflowInstanceIds(Lists.newArrayList(command.getProcessInstanceId())); + assertThat(commandMapper.selectList(null)).isEmpty(); + } + private boolean toTestQueryCommandPageBySlot(int masterCount, int thisMasterSlot) { Command command = createCommand(); Integer id = command.getId(); @@ -280,5 +290,4 @@ public class CommandMapperTest extends BaseDaoTest { return command; } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java index 3baa10b343..12cae5c53e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java @@ -252,12 +252,61 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { @Override public void kill() { try { - changeRunningSubprocessInstancesToStop(WorkflowExecutionStatus.READY_STOP); + doKillSubWorkflowInstances(); } catch (MasterTaskExecuteException e) { log.error("kill {} error", taskInstance.getName(), e); } } + private void doKillSubWorkflowInstances() throws MasterTaskExecuteException { + List existsSubProcessInstanceList = + subWorkflowService.getAllDynamicSubWorkflow(processInstance.getId(), taskInstance.getTaskCode()); + if (CollectionUtils.isEmpty(existsSubProcessInstanceList)) { + return; + } + + commandMapper.deleteByWorkflowInstanceIds( + existsSubProcessInstanceList.stream().map(ProcessInstance::getId).collect(Collectors.toList())); + + List runningSubProcessInstanceList = + subWorkflowService.filterRunningProcessInstances(existsSubProcessInstanceList); + doKillRunningSubWorkflowInstances(runningSubProcessInstanceList); + + List waitToRunProcessInstances = + subWorkflowService.filterWaitToRunProcessInstances(existsSubProcessInstanceList); + doKillWaitToRunSubWorkflowInstances(waitToRunProcessInstances); + + this.haveBeenCanceled = true; + } + + private void doKillRunningSubWorkflowInstances(List runningSubProcessInstanceList) throws MasterTaskExecuteException { + for (ProcessInstance subProcessInstance : runningSubProcessInstanceList) { + subProcessInstance.setState(WorkflowExecutionStatus.READY_STOP); + processInstanceDao.updateById(subProcessInstance); + if (subProcessInstance.getState().isFinished()) { + log.info("The process instance [{}] is finished, no need to stop", subProcessInstance.getId()); + continue; + } + try { + sendToSubProcess(taskExecutionContext, subProcessInstance); + log.info("Success send [{}] request to SubWorkflow's master: {}", WorkflowExecutionStatus.READY_STOP, + subProcessInstance.getHost()); + } catch (Exception e) { + throw new MasterTaskExecuteException( + String.format("Send stop request to SubWorkflow's master: %s failed", + subProcessInstance.getHost()), + e); + } + } + } + + private void doKillWaitToRunSubWorkflowInstances(List waitToRunWorkflowInstances) { + for (ProcessInstance subProcessInstance : waitToRunWorkflowInstances) { + subProcessInstance.setState(WorkflowExecutionStatus.STOP); + processInstanceDao.updateById(subProcessInstance); + } + } + private void changeRunningSubprocessInstancesToStop(WorkflowExecutionStatus stopStatus) throws MasterTaskExecuteException { this.haveBeenCanceled = true; List existsSubProcessInstanceList = From 446f6ba72b8347ad817d8f62cdd8d258ace7adf4 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 18:14:48 +0800 Subject: [PATCH 72/88] Fix auto create tennat concurrently will cause the task failed (#15909) --- .../common/utils/OSUtils.java | 25 ++++++++----------- .../utils/TaskExecutionContextUtils.java | 2 +- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index beca53c3fd..dbfcea2ed8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -183,11 +183,10 @@ public class OSUtils { * * @param userName user name */ - public static void createUserIfAbsent(String userName) { + public static synchronized void createUserIfAbsent(String userName) { // if not exists this user, then create if (!getUserList().contains(userName)) { - boolean isSuccess = createUser(userName); - log.info("create user {} {}", userName, isSuccess ? "success" : "fail"); + createUser(userName); } } @@ -197,13 +196,12 @@ public class OSUtils { * @param userName user name * @return true if creation was successful, otherwise false */ - public static boolean createUser(String userName) { + public static void createUser(String userName) { try { String userGroup = getGroup(); if (StringUtils.isEmpty(userGroup)) { - String errorLog = String.format("%s group does not exist for this operating system.", userGroup); - log.error(errorLog); - return false; + throw new UnsupportedOperationException( + "There is no userGroup exist cannot create tenant, please create userGroupFirst"); } if (SystemUtils.IS_OS_MAC) { createMacUser(userName, userGroup); @@ -212,18 +210,17 @@ public class OSUtils { } else { createLinuxUser(userName, userGroup); } - return true; + log.info("Create tenant {} under userGroup: {} success", userName, userGroup); } catch (Exception e) { - log.error(e.getMessage(), e); + throw new RuntimeException("Create tenant: {} failed", e); } - return false; } /** * create linux user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -237,7 +234,7 @@ public class OSUtils { /** * create mac user (Supports Mac OSX 10.10+) * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -256,7 +253,7 @@ public class OSUtils { /** * create windows user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -304,7 +301,7 @@ public class OSUtils { * get sudo command * * @param tenantCode tenantCode - * @param command command + * @param command command * @return result of sudo execute command */ public static String getSudoCmd(String tenantCode, String command) { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 3cda6ac099..f43b19c444 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -78,7 +78,7 @@ public class TaskExecutionContextUtils { throw ex; } catch (Exception ex) { throw new TaskException( - String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode())); + String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode()), ex); } } From d1135eabc7df9358fd210e423785d1067af64698 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 25 Apr 2024 19:39:56 +0800 Subject: [PATCH 73/88] [DSIP-34] Change required_approving_review_count to 2 (#15914) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index cf409dc6a3..84619447be 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,4 +49,4 @@ github: - "Mergeable: milestone-label-check" required_pull_request_reviews: dismiss_stale_reviews: true - required_approving_review_count: 1 + required_approving_review_count: 2 From b29965bdce2b5b83c1ffe237265e6f53f01e11cf Mon Sep 17 00:00:00 2001 From: DaqianLiao <360989637@qq.com> Date: Fri, 26 Apr 2024 11:24:05 +0800 Subject: [PATCH 74/88] [Fix] In updateWorkerNodes method, the workerNodeInfoWriteLock should be used. #15898 (#15903) Co-authored-by: answerliao --- .../server/master/registry/ServerNodeManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java index 258acd8f6e..f066f0403d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java @@ -245,7 +245,7 @@ public class ServerNodeManager implements InitializingBean { } private void updateWorkerNodes() { - workerGroupWriteLock.lock(); + workerNodeInfoWriteLock.lock(); try { Map workerNodeMaps = registryClient.getServerMaps(RegistryNodeType.WORKER); for (Map.Entry entry : workerNodeMaps.entrySet()) { @@ -254,7 +254,7 @@ public class ServerNodeManager implements InitializingBean { workerNodeInfo.put(nodeAddress, workerHeartBeat); } } finally { - workerGroupWriteLock.unlock(); + workerNodeInfoWriteLock.unlock(); } } From 7a55adeae9d9c5e8bc813d952322c8da1aabdc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E8=80=90?= <46134044+sdhzwc@users.noreply.github.com> Date: Sun, 28 Apr 2024 11:02:44 +0800 Subject: [PATCH 75/88] [Improvement-15919][datasource] Improvement datasource get name (#15920) --- .../AbstractDataSourceProcessor.java | 2 +- .../api/plugin/DataSourceClientProvider.java | 8 +-- .../AthenaDataSourceChannelFactory.java | 3 +- .../AzureSQLDataSourceChannelFactory.java | 3 +- .../ClickHouseDataSourceChannelFactory.java | 3 +- .../DamengDataSourceChannelFactory.java | 2 +- .../DatabendDataSourceChannelFactory.java | 3 +- .../DatabendDataSourceProcessorTest.java | 2 +- .../db2/DB2DataSourceChannelFactory.java | 3 +- .../doris/DorisDataSourceChannelFactory.java | 2 +- .../hana/HanaDataSourceChannelFactory.java | 3 +- .../hive/HiveDataSourceChannelFactory.java | 3 +- .../k8s/K8sDataSourceChannelFactory.java | 3 +- .../k8s/param/K8sDataSourceProcessor.java | 2 +- .../KyuubiDataSourceChannelFactory.java | 3 +- .../param/KyuubiDataSourceProcessorTest.java | 2 +- .../mysql/MySQLDataSourceChannelFactory.java | 3 +- .../OceanBaseDataSourceChannelFactory.java | 3 +- .../OracleDataSourceChannelFactory.java | 3 +- .../PostgreSQLDataSourceChannelFactory.java | 3 +- .../PrestoDataSourceChannelFactory.java | 3 +- .../RedshiftDataSourceChannelFactory.java | 3 +- .../SagemakerDataSourceChannelFactory.java | 3 +- .../param/SagemakerDataSourceProcessor.java | 2 +- .../SnowflakeDataSourceChannelFactory.java | 3 +- .../SnowflakeDataSourceProcessorTest.java | 2 +- .../spark/SparkDataSourceChannelFactory.java | 3 +- .../SQLServerDataSourceChannelFactory.java | 3 +- .../ssh/SSHDataSourceChannelFactory.java | 3 +- .../ssh/param/SSHDataSourceProcessor.java | 2 +- .../StarRocksDataSourceChannelFactory.java | 2 +- .../trino/TrinoDataSourceChannelFactory.java | 3 +- .../VerticaDataSourceChannelFactory.java | 3 +- .../ZeppelinDataSourceChannelFactory.java | 3 +- .../param/ZeppelinDataSourceProcessor.java | 2 +- .../dolphinscheduler/spi/enums/DbType.java | 62 ++++++++++--------- 36 files changed, 95 insertions(+), 66 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index 98222a2c5a..4acf531ddc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -118,7 +118,7 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getUser(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getUser(), PasswordUtils.encodePassword(baseConnectionParam.getPassword()), baseConnectionParam.getJdbcUrl()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java index 839a4c5d61..7223fe62a3 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java @@ -69,9 +69,9 @@ public class DataSourceClientProvider { String datasourceUniqueId = DataSourceUtils.getDatasourceUniqueId(baseConnectionParam, dbType); return POOLED_DATASOURCE_CLIENT_CACHE.get(datasourceUniqueId, () -> { Map dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap(); - DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp()); + DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getName()); if (null == dataSourceChannel) { - throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getDescp())); + throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getName())); } return dataSourceChannel.createPooledDataSourceClient(baseConnectionParam, dbType); }); @@ -85,9 +85,9 @@ public class DataSourceClientProvider { public static AdHocDataSourceClient getAdHocDataSourceClient(DbType dbType, ConnectionParam connectionParam) { BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; Map dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap(); - DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp()); + DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getName()); if (null == dataSourceChannel) { - throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getDescp())); + throw new RuntimeException(String.format("datasource plugin '%s' is not found", dbType.getName())); } return dataSourceChannel.createAdHocDataSourceClient(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java index 1b2ed367d0..b4759db39a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.athena; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,6 +33,6 @@ public class AthenaDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "athena"; + return DbType.ATHENA.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java index 5966848f33..2b8cdca973 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.azuresql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class AzureSQLDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "azuresql"; + return DbType.AZURESQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java index d756226522..77d0feb1d1 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.clickhouse; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class ClickHouseDataSourceChannelFactory implements DataSourceChannelFact @Override public String getName() { - return "clickhouse"; + return DbType.CLICKHOUSE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java index 945f6610c0..84ae080134 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/DamengDataSourceChannelFactory.java @@ -28,7 +28,7 @@ public class DamengDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return DbType.DAMENG.getDescp(); + return DbType.DAMENG.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java index 0ea40c3b13..3c86601dd7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.databend; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class DatabendDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "databend"; + return DbType.DATABEND.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java index f225a2fd3d..cb41c6562b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java @@ -151,7 +151,7 @@ public class DatabendDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(19, DbType.DATABEND.getCode()); - Assertions.assertEquals("databend", DbType.DATABEND.getDescp()); + Assertions.assertEquals("databend", DbType.DATABEND.getName()); Assertions.assertEquals(DbType.DATABEND, DbType.of(19)); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java index cda8a2e592..3bbae238ea 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.db2; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class DB2DataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "db2"; + return DbType.DB2.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java index d663c362f2..7180a6c6c2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java @@ -32,6 +32,6 @@ public class DorisDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return DbType.DORIS.getDescp(); + return DbType.DORIS.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java index 75aacebaff..91d275aab6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/HanaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hana; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class HanaDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "hana"; + return DbType.HANA.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java index 96ee007c8d..2caa4092dc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class HiveDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "hive"; + return DbType.HIVE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java index 03ec046de8..6a4428b47b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.k8s; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class K8sDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "k8s"; + return DbType.K8S.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java index 9e7342d433..fd3b49469f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java @@ -58,7 +58,7 @@ public class K8sDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { K8sConnectionParam baseConnectionParam = (K8sConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}", dbType.getDescp(), + return MessageFormat.format("{0}@{1}@{2}", dbType.getName(), PasswordUtils.encodePassword(baseConnectionParam.getKubeConfig()), baseConnectionParam.getNamespace()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java index 4c67a2098f..c60e74ccf8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/KyuubiDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.kyuubi; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class KyuubiDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "kyuubi"; + return DbType.KYUUBI.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java index 865565c5dc..a18ceb4216 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/test/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessorTest.java @@ -143,7 +143,7 @@ public class KyuubiDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(18, DbType.KYUUBI.getCode()); - Assertions.assertEquals("kyuubi", DbType.KYUUBI.getDescp()); + Assertions.assertEquals("kyuubi", DbType.KYUUBI.getName()); Assertions.assertEquals(DbType.KYUUBI, DbType.of(18)); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java index e57fc7e61d..adc3ec7946 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.mysql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class MySQLDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "mysql"; + return DbType.MYSQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java index a69d6b3ae5..13650679b0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/OceanBaseDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.oceanbase; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class OceanBaseDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "oceanbase"; + return DbType.OCEANBASE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java index dedbce4946..f63aff9a2b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.oracle; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class OracleDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "oracle"; + return DbType.ORACLE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java index 8aa6e566b7..e82a2e6860 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.postgresql; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class PostgreSQLDataSourceChannelFactory implements DataSourceChannelFact @Override public String getName() { - return "postgresql"; + return DbType.POSTGRESQL.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java index ed1292ffc9..76bf9d0808 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.presto; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class PrestoDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "presto"; + return DbType.PRESTO.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java index 25a587ae06..8c588f0b44 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,6 +33,6 @@ public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "redshift"; + return DbType.REDSHIFT.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java index 04ab93f36f..2457843614 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.sagemaker; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class SagemakerDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "sagemaker"; + return DbType.SAGEMAKER.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java index 4239f45e5c..7452ef8a14 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java @@ -57,7 +57,7 @@ public class SagemakerDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { SagemakerConnectionParam baseConnectionParam = (SagemakerConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), PasswordUtils.encodePassword(baseConnectionParam.getUserName()), PasswordUtils.encodePassword(baseConnectionParam.getPassword()), PasswordUtils.encodePassword(baseConnectionParam.getAwsRegion())); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java index 0d0c97ecd6..6bbfd7a6fb 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/SnowflakeDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.snowflake; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SnowflakeDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "snowflake"; + return DbType.SNOWFLAKE.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java index 54c5acf0f2..c60e70576f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/test/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDataSourceProcessorTest.java @@ -169,7 +169,7 @@ public class SnowflakeDataSourceProcessorTest { @Test public void testDbType() { Assertions.assertEquals(20, DbType.SNOWFLAKE.getCode()); - Assertions.assertEquals("snowflake", DbType.SNOWFLAKE.getDescp()); + Assertions.assertEquals("snowflake", DbType.SNOWFLAKE.getName()); Assertions.assertEquals(DbType.of(20), DbType.SNOWFLAKE); Assertions.assertEquals(DbType.ofName("SNOWFLAKE"), DbType.SNOWFLAKE); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java index dbda3da5bd..25f29ff21f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.spark; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SparkDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "spark"; + return DbType.SPARK.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java index e76f520d1e..f29cf6415e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.sqlserver; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SQLServerDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return "sqlserver"; + return DbType.SQLSERVER.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java index 3195432703..9742c97e9b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.ssh; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class SSHDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "ssh"; + return DbType.SSH.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java index 6bf0bed1b9..1916edba35 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/param/SSHDataSourceProcessor.java @@ -55,7 +55,7 @@ public class SSHDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { SSHConnectionParam baseConnectionParam = (SSHConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getHost(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getHost(), baseConnectionParam.getUser(), PasswordUtils.encodePassword(baseConnectionParam.getPassword())); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java index 50e2483952..82f78cff21 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/StarRocksDataSourceChannelFactory.java @@ -33,6 +33,6 @@ public class StarRocksDataSourceChannelFactory implements DataSourceChannelFacto @Override public String getName() { - return DbType.STARROCKS.getDescp(); + return DbType.STARROCKS.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java index 8c9605d791..36a3817fb0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/TrinoDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.trino; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class TrinoDataSourceChannelFactory implements DataSourceChannelFactory { @Override public String getName() { - return "trino"; + return DbType.TRINO.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java index b507a207b4..44e151f2f2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/VerticaDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.vertica; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -27,7 +28,7 @@ public class VerticaDataSourceChannelFactory implements DataSourceChannelFactory @Override public String getName() { - return "vertica"; + return DbType.VERTICA.getName(); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java index 692819cf78..559ee55836 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.datasource.zeppelin; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import com.google.auto.service.AutoService; @@ -32,7 +33,7 @@ public class ZeppelinDataSourceChannelFactory implements DataSourceChannelFactor @Override public String getName() { - return "zeppelin"; + return DbType.ZEPPELIN.getName(); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java index 92077275ad..88a913974e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java @@ -56,7 +56,7 @@ public class ZeppelinDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; - return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getRestEndpoint(), + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getName(), baseConnectionParam.getRestEndpoint(), baseConnectionParam.getUsername(), PasswordUtils.encodePassword(baseConnectionParam.getPassword())); } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index e7ebbeee0a..882b170e11 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -28,42 +28,44 @@ import com.google.common.base.Functions; public enum DbType { - MYSQL(0, "mysql"), - POSTGRESQL(1, "postgresql"), - HIVE(2, "hive"), - SPARK(3, "spark"), - CLICKHOUSE(4, "clickhouse"), - ORACLE(5, "oracle"), - SQLSERVER(6, "sqlserver"), - DB2(7, "db2"), - PRESTO(8, "presto"), - H2(9, "h2"), - REDSHIFT(10, "redshift"), - ATHENA(11, "athena"), - TRINO(12, "trino"), - STARROCKS(13, "starrocks"), - AZURESQL(14, "azuresql"), - DAMENG(15, "dameng"), - OCEANBASE(16, "oceanbase"), - SSH(17, "ssh"), - KYUUBI(18, "kyuubi"), - DATABEND(19, "databend"), - SNOWFLAKE(20, "snowflake"), - VERTICA(21, "vertica"), - HANA(22, "hana"), - DORIS(23, "doris"), - ZEPPELIN(24, "zeppelin"), - SAGEMAKER(25, "sagemaker"), + MYSQL(0, "mysql", "mysql"), + POSTGRESQL(1, "postgresql", "postgresql"), + HIVE(2, "hive", "hive"), + SPARK(3, "spark", "spark"), + CLICKHOUSE(4, "clickhouse", "clickhouse"), + ORACLE(5, "oracle", "oracle"), + SQLSERVER(6, "sqlserver", "sqlserver"), + DB2(7, "db2", "db2"), + PRESTO(8, "presto", "presto"), + H2(9, "h2", "h2"), + REDSHIFT(10, "redshift", "redshift"), + ATHENA(11, "athena", "athena"), + TRINO(12, "trino", "trino"), + STARROCKS(13, "starrocks", "starrocks"), + AZURESQL(14, "azuresql", "azuresql"), + DAMENG(15, "dameng", "dameng"), + OCEANBASE(16, "oceanbase", "oceanbase"), + SSH(17, "ssh", "ssh"), + KYUUBI(18, "kyuubi", "kyuubi"), + DATABEND(19, "databend", "databend"), + SNOWFLAKE(20, "snowflake", "snowflake"), + VERTICA(21, "vertica", "vertica"), + HANA(22, "hana", "hana"), + DORIS(23, "doris", "doris"), + ZEPPELIN(24, "zeppelin", "zeppelin"), + SAGEMAKER(25, "sagemaker", "sagemaker"), - K8S(26, "k8s"); + K8S(26, "k8s", "k8s"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); @EnumValue private final int code; + private final String name; private final String descp; - DbType(int code, String descp) { + DbType(int code, String name, String descp) { this.code = code; + this.name = name; this.descp = descp; } @@ -83,6 +85,10 @@ public enum DbType { return code; } + public String getName() { + return name; + } + public String getDescp() { return descp; } From 5a6c6c37f006291bc774ec44fe6d3d6793721195 Mon Sep 17 00:00:00 2001 From: calvin Date: Sun, 28 Apr 2024 18:47:20 +0800 Subject: [PATCH 76/88] [Improvement-15910][UI] Supposed to provide a default value for the custom parallelism when using the mode of parallel execution. (#15912) * worked out the issue * imrpove the parallism strategy * imrpove the parallism strategy * merge from dev --- .../src/locales/en_US/project.ts | 3 ++- .../src/locales/zh_CN/project.ts | 3 ++- .../definition/components/start-modal.tsx | 20 ++++++++++--------- .../definition/components/use-form.ts | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 7a39752526..5e5ee4b2a9 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -249,7 +249,8 @@ export default { delete_task_validate_dependent_tasks_desc: 'The downstream dependent tasks exists. You can not delete the task.', warning_delete_scheduler_dependent_tasks_desc: - 'The downstream dependent tasks exists. Are you sure to delete the scheduler?' + 'The downstream dependent tasks exists. Are you sure to delete the scheduler?', + warning_too_large_parallelism_number: 'The parallelism number is too large. It is better not to be over 10.' }, task: { on_line: 'Online', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 50e0a821ef..85de0122fb 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -246,7 +246,8 @@ export default { delete_task_validate_dependent_tasks_desc: '下游存在依赖，你不能删除该任务.', warning_delete_scheduler_dependent_tasks_desc: - '下游存在依赖, 删除定时可能会对下游任务产生影响. 你确定要删除该定时嘛?' + '下游存在依赖, 删除定时可能会对下游任务产生影响. 你确定要删除该定时嘛?', + warning_too_large_parallelism_number: '并行度设置太大了, 最好不要超过10.', }, task: { on_line: '线上', diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx index 872b515844..8a4b07b4bc 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/start-modal.tsx @@ -44,7 +44,8 @@ import { NSwitch, NCheckbox, NDatePicker, - NRadioButton + NRadioButton, + NInputNumber } from 'naive-ui' import { ArrowDownOutlined, @@ -75,7 +76,6 @@ export default defineComponent({ props, emits: ['update:show', 'update:row', 'updateList'], setup(props, ctx) { - const parallelismRef = ref(false) const { t } = useI18n() const route = useRoute() const { startState } = useForm() @@ -296,7 +296,6 @@ export default defineComponent({ return { t, showTaskDependType, - parallelismRef, hideModal, handleStart, generalWarningTypeListOptions, @@ -504,17 +503,20 @@ export default defineComponent({ 10 + } > - - {t('project.workflow.custom_parallelism')} - - )} diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts index 32b02a7880..33dd7ba241 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-form.ts @@ -66,7 +66,7 @@ export const useForm = () => { tenantCode: 'default', environmentCode: null, startParams: null, - expectedParallelismNumber: '', + expectedParallelismNumber: '2', dryRun: 0, testFlag: 0, version: null, From 647cbae4002c0ab3758d57827460ad7125a2c853 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 29 Apr 2024 16:14:23 +0800 Subject: [PATCH 77/88] [DSIP-32][Master] Add command fetcher strategy for master fetch command (#15900) --- .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - .../dolphinscheduler_env.sh | 1 - docs/docs/en/architecture/configuration.md | 4 +- .../en/guide/installation/pseudo-cluster.md | 1 - docs/docs/zh/architecture/configuration.md | 47 +++++----- .../zh/guide/installation/pseudo-cluster.md | 1 - .../dao/mapper/CommandMapper.java | 12 +-- .../dao/repository/BaseDao.java | 5 ++ .../dao/repository/CommandDao.java | 39 ++++++++ .../dolphinscheduler/dao/repository/IDao.java | 5 ++ .../dao/repository/impl/CommandDaoImpl.java | 41 +++++++++ .../dao/mapper/CommandMapper.xml | 6 +- .../dao/mapper/CommandMapperTest.java | 13 ++- .../repository/impl/CommandDaoImplTest.java | 88 +++++++++++++++++++ .../command/CommandFetcherConfiguration.java | 49 +++++++++++ .../master/command/ICommandFetcher.java | 36 ++++++++ .../command/IdSlotBasedCommandFetcher.java | 73 +++++++++++++++ .../master/config/CommandFetchStrategy.java | 63 +++++++++++++ .../server/master/config/MasterConfig.java | 12 +-- .../runner/MasterSchedulerBootstrap.java | 38 ++------ .../src/main/resources/application.yaml | 9 +- .../master/config/MasterConfigTest.java | 12 +++ .../src/test/resources/application.yaml | 9 +- .../service/command/CommandService.java | 11 --- .../service/command/CommandServiceImpl.java | 9 -- .../command/MessageServiceImplTest.java | 10 --- .../src/main/resources/application.yaml | 9 +- 29 files changed, 484 insertions(+), 122 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh index 58937e740c..8536eb0905 100755 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456 # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh index 671c70a5bb..f64e59b768 100755 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456 # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh index e7fd1b7204..29f8570319 100644 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=jdbc diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh index 1dbd63254e..6851716058 100644 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 13d8932943..fe0b7851ba 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -286,7 +286,6 @@ Location: `master-server/conf/application.yaml` | Parameters | Default value | Description | |-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | master.listen-port | 5678 | master listen port | -| master.fetch-command-num | 10 | the number of commands fetched by master | | master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel | | master.exec-threads | 100 | master execute thread number to limit process instances in parallel | | master.dispatch-task-number | 3 | master dispatch task number per batch | @@ -305,6 +304,9 @@ Location: `master-server/conf/application.yaml` | master.registry-disconnect-strategy.strategy | stop | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting | | master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely | | master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory | +| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db | +| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master | ### Worker Server related configuration diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index e63436f203..7a3b43b00e 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -123,7 +123,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 08fded19e0..d8d1d42d1e 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -281,29 +281,30 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 位置：`master-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -|-----------------------------------------------------------------------------|--------------|-----------------------------------------------------------------------------------------| -| master.listen-port | 5678 | master监听端口 | -| master.fetch-command-num | 10 | master拉取command数量 | -| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | -| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | -| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | -| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | -| master.max-heartbeat-interval | 10s | master最大心跳间隔 | -| master.task-commit-retry-times | 5 | 任务重试次数 | -| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | -| master.state-wheel-interval | 5 | 轮询检查状态时间 | -| master.server-load-protection.enabled | true | 是否开启系统保护策略 | -| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | -| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | -| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | -| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | -| master.failover-interval | 10 | failover间隔，单位为分钟 | -| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | -| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | -| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, | -| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | -| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| 参数 | 默认值 | 描述 | +|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------| +| master.listen-port | 5678 | master监听端口 | +| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | +| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | +| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | +| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | +| master.max-heartbeat-interval | 10s | master最大心跳间隔 | +| master.task-commit-retry-times | 5 | 任务重试次数 | +| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | +| master.state-wheel-interval | 5 | 轮询检查状态时间 | +| master.server-load-protection.enabled | true | 是否开启系统保护策略 | +| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | +| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | +| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | +| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | +| master.failover-interval | 10 | failover间隔，单位为分钟 | +| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | +| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | +| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | +| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| master.command-fetch-strategy.type | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | 数据库中t_ds_command的id自增步长 | +| master.command-fetch-strategy.config.fetch-size | 10 | master拉取command数量 | ## Worker Server相关配置 diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 13479e0d9e..a199167e04 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -118,7 +118,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index 9fb6643227..8c8314e7cc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -52,14 +52,10 @@ public interface CommandMapper extends BaseMapper { */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); - /** - * query command page by slot - * - * @return command list - */ - List queryCommandPageBySlot(@Param("limit") int limit, - @Param("masterCount") int masterCount, - @Param("thisMasterSlot") int thisMasterSlot); + List queryCommandByIdSlot(@Param("currentSlotIndex") int currentSlotIndex, + @Param("totalSlot") int totalSlot, + @Param("idStep") int idStep, + @Param("fetchNumber") int fetchNum); void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java index 2937957dbd..664b56ee47 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java @@ -56,6 +56,11 @@ public abstract class BaseDao> return mybatisMapper.selectBatchIds(ids); } + @Override + public List queryAll() { + return mybatisMapper.selectList(null); + } + @Override public List queryByCondition(ENTITY queryCondition) { if (queryCondition == null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java new file mode 100644 index 0000000000..daa52b8318 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.dao.entity.Command; + +import java.util.List; + +public interface CommandDao extends IDao { + + /** + * Query command by command id and server slot, return the command which match (commandId / step) %s totalSlot = currentSlotIndex + * + * @param currentSlotIndex current slot index + * @param totalSlot total slot number + * @param idStep id step in db + * @param fetchNum fetch number + * @return command list + */ + List queryCommandByIdSlot(int currentSlotIndex, + int totalSlot, + int idStep, + int fetchNum); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java index c566d9b904..ab77419600 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java @@ -41,6 +41,11 @@ public interface IDao { */ List queryByIds(Collection ids); + /** + * Query all entities. + */ + List queryAll(); + /** * Query the entity by condition. */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java new file mode 100644 index 0000000000..0b510d15b5 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import java.util.List; + +import org.springframework.stereotype.Repository; + +@Repository +public class CommandDaoImpl extends BaseDao implements CommandDao { + + public CommandDaoImpl(CommandMapper commandMapper) { + super(commandMapper); + } + + @Override + public List queryCommandByIdSlot(int currentSlotIndex, int totalSlot, int idStep, int fetchNum) { + return mybatisMapper.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchNum); + } + +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index 56db890ef0..16f7c05f25 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -40,12 +40,12 @@ limit #{limit} offset #{offset} - select * from t_ds_command - where id % #{masterCount} = #{thisMasterSlot} + where (id / #{idStep}) % #{totalSlot} = #{currentSlotIndex} order by process_instance_priority, id asc - limit #{limit} + limit #{fetchNumber} delete from t_ds_command diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 2d367e46e4..560b68754a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -187,7 +187,7 @@ public class CommandMapperTest extends BaseDaoTest { Command command = createCommand(); Integer id = command.getId(); boolean hit = id % masterCount == thisMasterSlot; - List commandList = commandMapper.queryCommandPageBySlot(1, masterCount, thisMasterSlot); + List commandList = commandMapper.queryCommandByIdSlot(thisMasterSlot, masterCount, 1, 1); if (hit) { Assertions.assertEquals(id, commandList.get(0).getId()); } else { @@ -201,8 +201,9 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command map - * @param count map count - * @param commandType comman type + * + * @param count map count + * @param commandType comman type * @param processDefinitionCode process definition code * @return command map */ @@ -223,7 +224,8 @@ public class CommandMapperTest extends BaseDaoTest { } /** - * create process definition + * create process definition + * * @return process definition */ private ProcessDefinition createProcessDefinition() { @@ -243,6 +245,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command map + * * @param count map count * @return command map */ @@ -258,6 +261,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command + * * @return */ private Command createCommand() { @@ -266,6 +270,7 @@ public class CommandMapperTest extends BaseDaoTest { /** * create command + * * @return Command */ private Command createCommand(CommandType commandType, long processDefinitionCode) { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java new file mode 100644 index 0000000000..85867ef3b5 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import org.apache.commons.lang3.RandomUtils; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class CommandDaoImplTest extends BaseDaoTest { + + @Autowired + private CommandDao commandDao; + + @Test + void fetchCommandByIdSlot() { + int commandSize = RandomUtils.nextInt(1, 1000); + for (int i = 0; i < commandSize; i++) { + createCommand(CommandType.START_PROCESS, 0); + } + int totalSlot = RandomUtils.nextInt(1, 10); + int currentSlotIndex = RandomUtils.nextInt(0, totalSlot); + int fetchSize = RandomUtils.nextInt(10, 100); + for (int i = 1; i < 5; i++) { + int idStep = i; + List commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize); + assertThat(commands.size()).isGreaterThan(0); + assertThat(commands.size()) + .isEqualTo(commandDao.queryAll() + .stream() + .filter(command -> (command.getId() / idStep) % totalSlot == currentSlotIndex) + .limit(fetchSize) + .count()); + + } + + } + + private void createCommand(CommandType commandType, int processDefinitionCode) { + Command command = new Command(); + command.setCommandType(commandType); + command.setProcessDefinitionCode(processDefinitionCode); + command.setExecutorId(4); + command.setCommandParam("test command param"); + command.setTaskDependType(TaskDependType.TASK_ONLY); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00")); + command.setProcessInstancePriority(Priority.MEDIUM); + command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP); + command.setProcessInstanceId(0); + command.setProcessDefinitionVersion(0); + commandDao.insert(command); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java new file mode 100644 index 0000000000..4a4d3c1efc --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/CommandFetcherConfiguration.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.server.master.config.CommandFetchStrategy; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class CommandFetcherConfiguration { + + @Bean + public ICommandFetcher commandFetcher(MasterConfig masterConfig, + MasterSlotManager masterSlotManager, + CommandDao commandDao) { + CommandFetchStrategy commandFetchStrategy = + checkNotNull(masterConfig.getCommandFetchStrategy(), "command fetch strategy is null"); + switch (commandFetchStrategy.getType()) { + case ID_SLOT_BASED: + CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig = + (CommandFetchStrategy.IdSlotBasedFetchConfig) commandFetchStrategy.getConfig(); + return new IdSlotBasedCommandFetcher(idSlotBasedFetchConfig, masterSlotManager, commandDao); + default: + throw new IllegalArgumentException( + "unsupported command fetch strategy type: " + commandFetchStrategy.getType()); + } + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java new file mode 100644 index 0000000000..c315a9b294 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/ICommandFetcher.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import org.apache.dolphinscheduler.dao.entity.Command; + +import java.util.List; + +/** + * The command fetcher used to fetch commands + */ +public interface ICommandFetcher { + + /** + * Fetch commands + * + * @return command list which need to be handled + */ + List fetchCommands(); + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java new file mode 100644 index 0000000000..a417820093 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/command/IdSlotBasedCommandFetcher.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.command; + +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.server.master.config.CommandFetchStrategy; +import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; +import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; + +import java.util.Collections; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +/** + * The command fetcher which is fetch commands by command id and slot. + */ +@Slf4j +public class IdSlotBasedCommandFetcher implements ICommandFetcher { + + private final CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig; + + private final CommandDao commandDao; + + private final MasterSlotManager masterSlotManager; + + public IdSlotBasedCommandFetcher(CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig, + MasterSlotManager masterSlotManager, + CommandDao commandDao) { + this.idSlotBasedFetchConfig = idSlotBasedFetchConfig; + this.masterSlotManager = masterSlotManager; + this.commandDao = commandDao; + } + + @Override + public List fetchCommands() { + long scheduleStartTime = System.currentTimeMillis(); + int currentSlotIndex = masterSlotManager.getSlot(); + int totalSlot = masterSlotManager.getMasterSize(); + if (totalSlot <= 0 || currentSlotIndex < 0) { + log.warn("Slot is validated, current master slots: {}, the current slot index is {}", totalSlot, + currentSlotIndex); + return Collections.emptyList(); + } + List commands = commandDao.queryCommandByIdSlot( + currentSlotIndex, + totalSlot, + idSlotBasedFetchConfig.getIdStep(), + idSlotBasedFetchConfig.getFetchSize()); + long cost = System.currentTimeMillis() - scheduleStartTime; + log.info("Fetch commands: {} success, cost: {}ms, totalSlot: {}, currentSlotIndex: {}", commands.size(), cost, + totalSlot, currentSlotIndex); + ProcessInstanceMetrics.recordCommandQueryTime(cost); + return commands; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java new file mode 100644 index 0000000000..e61941677c --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/CommandFetchStrategy.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.config; + +import lombok.Data; + +import org.springframework.validation.Errors; + +@Data +public class CommandFetchStrategy { + + private CommandFetchStrategyType type = CommandFetchStrategyType.ID_SLOT_BASED; + + private CommandFetchConfig config = new IdSlotBasedFetchConfig(); + + public void validate(Errors errors) { + config.validate(errors); + } + + public enum CommandFetchStrategyType { + ID_SLOT_BASED, + ; + } + + public interface CommandFetchConfig { + + void validate(Errors errors); + + } + + @Data + public static class IdSlotBasedFetchConfig implements CommandFetchConfig { + + private int idStep = 1; + private int fetchSize = 10; + + @Override + public void validate(Errors errors) { + if (idStep <= 0) { + errors.rejectValue("step", null, "step must be greater than 0"); + } + if (fetchSize <= 0) { + errors.rejectValue("fetchSize", null, "fetchSize must be greater than 0"); + } + } + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index 02c0dcb819..20d3cccef3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -48,10 +48,6 @@ public class MasterConfig implements Validator { * The master RPC server listen port. */ private int listenPort = 5678; - /** - * The max batch size used to fetch command from database. - */ - private int fetchCommandNum = 10; /** * The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum. */ @@ -98,6 +94,8 @@ public class MasterConfig implements Validator { private Duration workerGroupRefreshInterval = Duration.ofSeconds(10L); + private CommandFetchStrategy commandFetchStrategy = new CommandFetchStrategy(); + // ip:listenPort private String masterAddress; @@ -115,9 +113,6 @@ public class MasterConfig implements Validator { if (masterConfig.getListenPort() <= 0) { errors.rejectValue("listen-port", null, "is invalidated"); } - if (masterConfig.getFetchCommandNum() <= 0) { - errors.rejectValue("fetch-command-num", null, "should be a positive value"); - } if (masterConfig.getPreExecThreads() <= 0) { errors.rejectValue("per-exec-threads", null, "should be a positive value"); } @@ -149,6 +144,7 @@ public class MasterConfig implements Validator { if (StringUtils.isEmpty(masterConfig.getMasterAddress())) { masterConfig.setMasterAddress(NetUtils.getAddr(masterConfig.getListenPort())); } + commandFetchStrategy.validate(errors); masterConfig.setMasterRegistryPath( RegistryNodeType.MASTER.getRegistryPath() + "/" + masterConfig.getMasterAddress()); @@ -159,7 +155,6 @@ public class MasterConfig implements Validator { String config = "\n****************************Master Configuration**************************************" + "\n listen-port -> " + listenPort + - "\n fetch-command-num -> " + fetchCommandNum + "\n pre-exec-threads -> " + preExecThreads + "\n exec-threads -> " + execThreads + "\n dispatch-task-number -> " + dispatchTaskNumber + @@ -175,6 +170,7 @@ public class MasterConfig implements Validator { "\n master-address -> " + masterAddress + "\n master-registry-path: " + masterRegistryPath + "\n worker-group-refresh-interval: " + workerGroupRefreshInterval + + "\n command-fetch-strategy: " + commandFetchStrategy + "\n****************************Master Configuration**************************************"; log.info(config); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java index 2fddd94384..c1b5d0ffab 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java @@ -26,21 +26,18 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; +import org.apache.dolphinscheduler.server.master.command.ICommandFetcher; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection; import org.apache.dolphinscheduler.server.master.event.WorkflowEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.event.WorkflowEventType; -import org.apache.dolphinscheduler.server.master.exception.MasterException; import org.apache.dolphinscheduler.server.master.exception.WorkflowCreateException; import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; -import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; -import org.apache.dolphinscheduler.server.master.registry.MasterSlotManager; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.commons.collections4.CollectionUtils; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -56,6 +53,9 @@ import org.springframework.stereotype.Service; @Slf4j public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCloseable { + @Autowired + private ICommandFetcher commandFetcher; + @Autowired private CommandService commandService; @@ -74,9 +74,6 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl @Autowired private WorkflowEventLooper workflowEventLooper; - @Autowired - private MasterSlotManager masterSlotManager; - @Autowired private MasterTaskExecutorBootstrap masterTaskExecutorBootstrap; @@ -125,7 +122,7 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } - List commands = findCommands(); + List commands = commandFetcher.fetchCommands(); if (CollectionUtils.isEmpty(commands)) { // indicate that no command ,sleep for 1s Thread.sleep(Constants.SLEEP_TIME_MILLIS); @@ -170,29 +167,4 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl } } - private List findCommands() throws MasterException { - try { - long scheduleStartTime = System.currentTimeMillis(); - int thisMasterSlot = masterSlotManager.getSlot(); - int masterCount = masterSlotManager.getMasterSize(); - if (masterCount <= 0) { - log.warn("Master count: {} is invalid, the current slot: {}", masterCount, thisMasterSlot); - return Collections.emptyList(); - } - int pageSize = masterConfig.getFetchCommandNum(); - final List result = - commandService.findCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - if (CollectionUtils.isNotEmpty(result)) { - long cost = System.currentTimeMillis() - scheduleStartTime; - log.info( - "Master schedule bootstrap loop command success, fetch command size: {}, cost: {}ms, current slot: {}, total slot size: {}", - result.size(), cost, thisMasterSlot, masterCount); - ProcessInstanceMetrics.recordCommandQueryTime(cost); - } - return result; - } catch (Exception ex) { - throw new MasterException("Master loop command from database error", ex); - } - } - } diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index f18c6ef61d..17b1e41a71 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -83,8 +83,6 @@ registry: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -121,6 +119,13 @@ master: # The max waiting time to reconnect to registry if you set the strategy to waiting max-waiting-time: 100s worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 server: port: 5679 diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java index faab44cf85..9d26aa81f4 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.config; +import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -47,6 +48,17 @@ public class MasterConfigTest { assertEquals(0.77, serverLoadProtection.getMaxJvmCpuUsagePercentageThresholds()); assertEquals(0.77, serverLoadProtection.getMaxSystemMemoryUsagePercentageThresholds()); assertEquals(0.77, serverLoadProtection.getMaxDiskUsagePercentageThresholds()); + } + @Test + public void getCommandFetchStrategy() { + CommandFetchStrategy commandFetchStrategy = masterConfig.getCommandFetchStrategy(); + assertThat(commandFetchStrategy.getType()) + .isEqualTo(CommandFetchStrategy.CommandFetchStrategyType.ID_SLOT_BASED); + + CommandFetchStrategy.IdSlotBasedFetchConfig idSlotBasedFetchConfig = + (CommandFetchStrategy.IdSlotBasedFetchConfig) commandFetchStrategy.getConfig(); + assertThat(idSlotBasedFetchConfig.getIdStep()).isEqualTo(3); + assertThat(idSlotBasedFetchConfig.getFetchSize()).isEqualTo(11); } } diff --git a/dolphinscheduler-master/src/test/resources/application.yaml b/dolphinscheduler-master/src/test/resources/application.yaml index f4827d4b3c..15f9199609 100644 --- a/dolphinscheduler-master/src/test/resources/application.yaml +++ b/dolphinscheduler-master/src/test/resources/application.yaml @@ -89,8 +89,6 @@ registry: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -127,6 +125,13 @@ master: # The max waiting time to reconnect to registry if you set the strategy to waiting max-waiting-time: 100s worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 3 + # master fetch command num + fetch-size: 11 server: port: 5679 diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java index cff73c503f..43b81c4e5c 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandService.java @@ -22,8 +22,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import java.util.List; - /** * Command Service */ @@ -44,15 +42,6 @@ public interface CommandService { */ int createCommand(Command command); - /** - * Get command page - * @param pageSize page size - * @param masterCount master count - * @param thisMasterSlot master slot - * @return command page - */ - List findCommandPageBySlot(int pageSize, int masterCount, int thisMasterSlot); - /** * check the input command exists in queue list * diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java index 483899446b..ee833a80b0 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/command/CommandServiceImpl.java @@ -57,7 +57,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; import io.micrometer.core.annotation.Counted; /** @@ -107,14 +106,6 @@ public class CommandServiceImpl implements CommandService { return result; } - @Override - public List findCommandPageBySlot(int pageSize, int masterCount, int thisMasterSlot) { - if (masterCount <= 0) { - return Lists.newArrayList(); - } - return commandMapper.queryCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - } - @Override public boolean verifyIsNeedCreateCommand(Command command) { boolean isNeedCreate = true; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java index 0cde76bdfe..f60320fc63 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/command/MessageServiceImplTest.java @@ -214,14 +214,4 @@ class MessageServiceImplTest { Mockito.verify(commandMapper, Mockito.times(1)).insert(command); } - @Test - public void testFindCommandPageBySlot() { - int pageSize = 1; - int masterCount = 0; - int thisMasterSlot = 2; - List commandList = - commandService.findCommandPageBySlot(pageSize, masterCount, thisMasterSlot); - Assertions.assertEquals(0, commandList.size()); - } - } diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 5122eea2a1..6757718929 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -160,8 +160,6 @@ casdoor: master: listen-port: 5678 - # master fetch command num - fetch-command-num: 10 # master prepare execute thread number to limit handle commands in parallel pre-exec-threads: 10 # master execute thread number to limit process instances in parallel @@ -192,6 +190,13 @@ master: # kill yarn/k8s application when failover taskInstance, default true kill-application-when-task-failover: true worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 worker: # worker listener port From 0a11cd21bd0e73b52d4e68dae2f32519031e2e50 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 29 Apr 2024 18:07:05 +0800 Subject: [PATCH 78/88] Fix jar path is not correct in java task (#15906) --- .../task/api/resource/ResourceContext.java | 1 - .../plugin/task/hivecli/HiveCliTaskTest.java | 4 +- .../plugin/task/java/JavaTask.java | 41 ++----------------- .../plugin/task/java/JavaTaskTest.java | 32 ++++++++------- .../utils/TaskExecutionContextUtils.java | 1 - 5 files changed, 23 insertions(+), 56 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java index 687d1aeb95..f90b526902 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java @@ -60,7 +60,6 @@ public class ResourceContext { public static class ResourceItem { private String resourceAbsolutePathInStorage; - private String resourceRelativePath; private String resourceAbsolutePathInLocal; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java index 824ad49f89..b4136af3c3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java @@ -65,7 +65,7 @@ public class HiveCliTaskTest { } @Test - public void hiveCliTaskExecuteSqlFromScript() throws Exception { + public void hiveCliTaskExecuteSqlFromScript() { String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters(); HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); hiveCliTask.init(); @@ -78,7 +78,7 @@ public class HiveCliTaskTest { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskParams(hiveCliTaskParameters); ResourceContext resourceContext = new ResourceContext(); - resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql", + resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "/sql_tasks/hive_task.sql")); taskExecutionContext.setResourceContext(resourceContext); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index 179b50c35c..fc23260345 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -88,6 +88,7 @@ public class JavaTask extends AbstractTask { /** * Initializes a Java task + * * @return void **/ @Override @@ -178,7 +179,8 @@ public class JavaTask extends AbstractTask { **/ protected String buildJarCommand() { ResourceContext resourceContext = taskRequest.getResourceContext(); - String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName()) + String mainJarAbsolutePathInLocal = resourceContext + .getResourceItem(javaParameters.getMainJar().getResourceName()) .getResourceAbsolutePathInLocal(); StringBuilder builder = new StringBuilder(); builder.append(getJavaCommandPath()) @@ -186,7 +188,7 @@ public class JavaTask extends AbstractTask { .append(buildResourcePath()).append(" ") .append("-jar").append(" ") .append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR) - .append(mainJarName).append(" ") + .append(mainJarAbsolutePathInLocal).append(" ") .append(javaParameters.getMainArgs().trim()).append(" ") .append(javaParameters.getJvmArgs().trim()); return builder.toString(); @@ -207,39 +209,6 @@ public class JavaTask extends AbstractTask { return javaParameters; } - /** - * Replaces placeholders such as local variables in source files - * - * @param rawScript - * @return String - * @throws StringIndexOutOfBoundsException - */ - protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException { - int len = "${setShareVar(${".length(); - - int scriptStart = 0; - while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { - int start = -1; - int end = rawScript.indexOf('}', scriptStart + len); - String prop = rawScript.substring(scriptStart + len, end); - - start = rawScript.indexOf(',', end); - end = rawScript.indexOf(')', start); - - String value = rawScript.substring(start + 1, end); - - start = rawScript.indexOf('}', start) + 1; - end = rawScript.length(); - - String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); - - rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); - - scriptStart += replaceScript.length(); - } - return rawScript; - } - /** * Creates a Java source file when it does not exist * @@ -290,8 +259,6 @@ public class JavaTask extends AbstractTask { for (ResourceInfo info : javaParameters.getResourceFilesList()) { builder.append(JavaConstants.PATH_SEPARATOR); builder - .append(taskRequest.getExecutePath()) - .append(FOLDER_SEPARATOR) .append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal()); } return builder.toString(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java index 55756241ce..c829415326 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.java; +import static com.google.common.truth.Truth.assertThat; import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; @@ -34,7 +35,6 @@ import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExis import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; -import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; @@ -82,10 +82,10 @@ public class JavaTaskTest { **/ @Test public void buildJarCommand() { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; JavaTask javaTask = runJarType(); - Assertions.assertEquals(javaTask.buildJarCommand(), homeBinPath - + "java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); + assertThat(javaTask.buildJarCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); } /** @@ -101,14 +101,13 @@ public class JavaTaskTest { Assertions.assertEquals("JavaTaskTest", publicClassName); String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); try { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(homeBinPath - + "javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java", - javaTask.buildJavaCompileCommand(sourceCode)); + assertThat(javaTask.buildJavaCompileCommand(sourceCode)) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java"); } finally { Path path = Paths.get(fileName); if (Files.exists(path)) { @@ -121,26 +120,29 @@ public class JavaTaskTest { /** * Construct java to run the command * - * @return void + * @return void **/ @Test public void buildJavaCommand() throws Exception { - String wantJavaCommand = - "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"; JavaTask javaTask = runJavaType(); String sourceCode = javaTask.buildJavaSourceContent(); String publicClassName = javaTask.getPublicClassName(sourceCode); + Assertions.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(wantJavaCommand, javaTask.buildJavaCommand()); + assertThat(javaTask.buildJavaCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"); } /** * There is no exception to overwriting the Java source file + * * @return void * @throws IOException **/ @@ -259,8 +261,8 @@ public class JavaTaskTest { resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); - resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); - resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInLocal("JavaTaskTest.java"); ResourceContext resourceContext = new ResourceContext(); resourceContext.addResourceItem(resourceItem1); @@ -275,7 +277,7 @@ public class JavaTaskTest { /** * The Java task to construct the jar run mode * - * @return JavaTask + * @return JavaTask **/ private JavaTask runJarType() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index f43b19c444..8d83dde593 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -151,7 +151,6 @@ public class TaskExecutionContextUtils { } ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder() .resourceAbsolutePathInStorage(resourceAbsolutePathInStorage) - .resourceRelativePath(resourceRelativePath) .resourceAbsolutePathInLocal(resourceAbsolutePathInLocal) .build(); resourceContext.addResourceItem(resourceItem); From ebcdaeb9ac125a13b74ec3f057693816a6758426 Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 29 Apr 2024 21:03:01 +0800 Subject: [PATCH 79/88] [TEST] increase coverage of project preference service test (#15939) --- .../service/ProjectPreferenceServiceTest.java | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java index 530c15d48e..7a74a7c265 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java @@ -60,28 +60,65 @@ public class ProjectPreferenceServiceTest { public void testUpdateProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + + // success Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); - Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(1); - Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operatation fail + Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.CREATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + + // success + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operation fail + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); } @Test public void testQueryProjectPreferenceByProjectCode() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + // PROJECT_PARAMETER_NOT_EXISTS Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), Mockito.any())).thenReturn(true); Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); - Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); // SUCCESS @@ -94,14 +131,29 @@ public class ProjectPreferenceServiceTest { public void testEnableProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); + // success Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); - Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + // db operation fail + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_STATE_ERROR.getCode(), result.getCode()); } private User getGeneralUser() { From fa6ea8ba7b06e21177114058b1f1c32fe005a60a Mon Sep 17 00:00:00 2001 From: Evan Sun Date: Mon, 6 May 2024 10:02:04 +0800 Subject: [PATCH 80/88] [TEST] increase coverage of project workergroup relation service (#15944) Co-authored-by: abzymeinsjtu --- ...ProjectWorkerGroupRelationServiceImpl.java | 12 +- ...ProjectWorkerGroupRelationServiceTest.java | 108 ++++++++++++++---- 2 files changed, 96 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java index 15cbcbb7fa..31dc113dbb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectWorkerGroupMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; @@ -38,6 +39,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -113,23 +115,25 @@ public class ProjectWorkerGroupRelationServiceImpl extends BaseServiceImpl } Set workerGroupNames = - workerGroupMapper.queryAllWorkerGroup().stream().map(item -> item.getName()).collect( + workerGroupMapper.queryAllWorkerGroup().stream().map(WorkerGroup::getName).collect( Collectors.toSet()); workerGroupNames.add(Constants.DEFAULT_WORKER_GROUP); - Set assignedWorkerGroupNames = workerGroups.stream().collect(Collectors.toSet()); + Set assignedWorkerGroupNames = new HashSet<>(workerGroups); Set difference = SetUtils.difference(assignedWorkerGroupNames, workerGroupNames); - if (difference.size() > 0) { + if (!difference.isEmpty()) { putMsg(result, Status.WORKER_GROUP_NOT_EXIST, difference.toString()); return result; } Set projectWorkerGroupNames = projectWorkerGroupMapper.selectList(new QueryWrapper() .lambda() - .eq(ProjectWorkerGroup::getProjectCode, projectCode)).stream().map(item -> item.getWorkerGroup()) + .eq(ProjectWorkerGroup::getProjectCode, projectCode)) + .stream() + .map(ProjectWorkerGroup::getWorkerGroup) .collect(Collectors.toSet()); difference = SetUtils.difference(projectWorkerGroupNames, assignedWorkerGroupNames); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java index a8e8b8beb2..6f6b7ca2d6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java @@ -17,13 +17,17 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getAdminUser; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectWorkerGroupRelationServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -33,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -77,27 +82,87 @@ public class ProjectWorkerGroupRelationServiceTest { @Test public void testAssignWorkerGroupsToProject() { + User generalUser = getGeneralUser(); User loginUser = getAdminUser(); - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); - Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + // no permission + Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(generalUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), result.getCode()); + + // project code is null + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, null, getWorkerGroups()); Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + // worker group is empty + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + Collections.emptyList()); + Assertions.assertEquals(Status.WORKER_GROUP_TO_PROJECT_IS_EMPTY.getCode(), result.getCode()); + + // project not exists + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + + // worker group not exists WorkerGroup workerGroup = new WorkerGroup(); workerGroup.setName("test"); Mockito.when(projectMapper.queryByCode(Mockito.anyLong())).thenReturn(getProject()); - Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Lists.newArrayList(workerGroup)); + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getDiffWorkerGroups()); + Assertions.assertEquals(Status.WORKER_GROUP_NOT_EXIST.getCode(), result.getCode()); + + // db insertion fail + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // success Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(1); result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, getWorkerGroups()); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // success when there is diff between current wg and assigned wg + Mockito.when(projectWorkerGroupMapper.selectList(Mockito.any())) + .thenReturn(Collections.singletonList(getDiffProjectWorkerGroup())); + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(1); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // db deletion fail + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // fail when wg is referenced by task definition + Mockito.when(taskDefinitionMapper.queryAllDefinitionList(Mockito.anyLong())) + .thenReturn(Collections.singletonList(getTaskDefinitionWithDiffWorkerGroup())); + AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); } @Test public void testQueryWorkerGroupsByProject() { + // no permission + Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) + .thenReturn(false); + Map result = + projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + + Assertions.assertTrue(result.isEmpty()); + + // success Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) .thenReturn(true); @@ -113,8 +178,7 @@ public class ProjectWorkerGroupRelationServiceTest { Mockito.when(scheduleMapper.querySchedulerListByProjectName(Mockito.any())) .thenReturn(Lists.newArrayList()); - Map result = - projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + result = projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); ProjectWorkerGroup[] actualValue = ((List) result.get(Constants.DATA_LIST)).toArray(new ProjectWorkerGroup[0]); @@ -126,20 +190,8 @@ public class ProjectWorkerGroupRelationServiceTest { return Lists.newArrayList("default"); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; - } - - private User getAdminUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.ADMIN_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + private List getDiffWorkerGroups() { + return Lists.newArrayList("default", "new"); } private Project getProject() { @@ -158,4 +210,20 @@ public class ProjectWorkerGroupRelationServiceTest { projectWorkerGroup.setWorkerGroup("default"); return projectWorkerGroup; } + + private ProjectWorkerGroup getDiffProjectWorkerGroup() { + ProjectWorkerGroup projectWorkerGroup = new ProjectWorkerGroup(); + projectWorkerGroup.setId(2); + projectWorkerGroup.setProjectCode(projectCode); + projectWorkerGroup.setWorkerGroup("new"); + return projectWorkerGroup; + } + + private TaskDefinition getTaskDefinitionWithDiffWorkerGroup() { + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setProjectCode(projectCode); + taskDefinition.setId(1); + taskDefinition.setWorkerGroup("new"); + return taskDefinition; + } } From d218b021e7e045e7982a3c91ea7b1b1491d21d38 Mon Sep 17 00:00:00 2001 From: JohnHuang Date: Mon, 6 May 2024 11:33:16 +0800 Subject: [PATCH 81/88] [DSIP-25][Remote Logging] Split remote logging configuration (#15826) Co-authored-by: Rick Cheng --- .../dolphinscheduler-alert-server.xml | 1 + .../assembly/dolphinscheduler-api-server.xml | 1 + .../ImmutablePriorityPropertyDelegate.java | 33 ++++++-- .../config/ImmutablePropertyDelegate.java | 2 +- .../common/config/ImmutableYamlDelegate.java | 82 +++++++++++++++++++ .../common/constants/Constants.java | 5 +- .../common/utils/PropertyUtils.java | 10 ++- .../src/main/resources/remote-logging.yaml | 61 ++++++++++++++ ...ImmutablePriorityPropertyDelegateTest.java | 5 +- .../src/test/resources/remote-logging.yaml | 61 ++++++++++++++ .../dolphinscheduler-master-server.xml | 1 + .../dolphinscheduler-worker-server.xml | 1 + 12 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java create mode 100644 dolphinscheduler-common/src/main/resources/remote-logging.yaml create mode 100644 dolphinscheduler-common/src/test/resources/remote-logging.yaml diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 6f39096221..bf28193b3f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -52,6 +52,7 @@ ${basedir}/../../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index 77b2f54c64..c9fdab4d51 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java index 742e745fe4..620f74ef95 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java @@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j; * This class will get the property by the priority of the following: env > jvm > properties. */ @Slf4j -public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate { +public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate { private static final Map>> configValueMap = new ConcurrentHashMap<>(); - public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) { - super(propertyAbsolutePath); + private ImmutablePropertyDelegate immutablePropertyDelegate; + + private ImmutableYamlDelegate immutableYamlDelegate; + + public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate, + ImmutableYamlDelegate immutableYamlDelegate) { + this.immutablePropertyDelegate = immutablePropertyDelegate; + this.immutableYamlDelegate = immutableYamlDelegate; } @Override @@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate return value; } value = getConfigValueFromProperties(key); + if (value.isPresent()) { + log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + k, value.get().getActualKey(), value.get().getValue()); + return value; + } + value = getConfigValueFromYaml(key); value.ifPresent( - stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}", k, stringConfigValue.getActualKey(), stringConfigValue.getValue())); return value; }); @@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate @Override public Set getPropertyKeys() { Set propertyKeys = new HashSet<>(); - propertyKeys.addAll(super.getPropertyKeys()); + propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys()); + propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys()); propertyKeys.addAll(System.getProperties().stringPropertyNames()); propertyKeys.addAll(System.getenv().keySet()); return propertyKeys; @@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate } private Optional> getConfigValueFromProperties(String key) { - String value = super.get(key); + String value = this.immutablePropertyDelegate.get(key); + if (value != null) { + return Optional.of(ConfigValue.fromProperties(key, value)); + } + return Optional.empty(); + } + + private Optional> getConfigValueFromYaml(String key) { + String value = this.immutableYamlDelegate.get(key); if (value != null) { return Optional.of(ConfigValue.fromProperties(key, value)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java index b58735afb0..4a0c192210 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java @@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate { } catch (IOException e) { log.error("Load property: {} error, please check if the file exist under classpath", propertyAbsolutePath, e); - System.exit(1); + throw new RuntimeException(e); } } printProperties(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java new file mode 100644 index 0000000000..5806a20fd7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.config; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.InputStreamResource; + +@Slf4j +public class ImmutableYamlDelegate implements IPropertyDelegate { + + private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml"; + + private final Properties properties; + + public ImmutableYamlDelegate() { + this(REMOTE_LOGGING_YAML_NAME); + } + + public ImmutableYamlDelegate(String... yamlAbsolutePath) { + properties = new Properties(); + // read from classpath + for (String fileName : yamlAbsolutePath) { + try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { + YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); + factory.setResources(new InputStreamResource(fis)); + factory.afterPropertiesSet(); + Properties subProperties = factory.getObject(); + properties.putAll(subProperties); + } catch (IOException e) { + log.error("Load property: {} error, please check if the file exist under classpath", + yamlAbsolutePath, e); + throw new RuntimeException(e); + } + } + printProperties(); + } + + public ImmutableYamlDelegate(Properties properties) { + this.properties = properties; + } + + @Override + public String get(String key) { + return properties.getProperty(key); + } + + @Override + public String get(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } + + @Override + public Set getPropertyKeys() { + return properties.stringPropertyNames(); + } + + private void printProperties() { + properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v)); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index ebf668a312..cc07accc9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -35,6 +35,8 @@ public final class Constants { */ public static final String COMMON_PROPERTIES_PATH = "/common.properties"; + public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; + public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; public static final String FORMAT_S_S_COLON = "%s:%s"; @@ -683,9 +685,6 @@ public final class Constants { public static final Integer QUERY_ALL_ON_WORKFLOW = 2; public static final Integer QUERY_ALL_ON_TASK = 3; - /** - * remote logging - */ public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable"; public static final String REMOTE_LOGGING_TARGET = "remote.logging.target"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8289b14479..82d4de9599 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; -import org.apache.dolphinscheduler.common.config.IPropertyDelegate; import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate; import java.util.HashMap; import java.util.Map; @@ -37,8 +39,10 @@ import com.google.common.base.Strings; public class PropertyUtils { // todo: add another implementation for zookeeper/etcd/consul/xx - private static final IPropertyDelegate propertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + private final ImmutablePriorityPropertyDelegate propertyDelegate = + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml new file mode 100644 index 0000000000..2cb48750a4 --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java index efba923a5a..6333250492 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config; import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test; class ImmutablePriorityPropertyDelegateTest { private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); @Test void getOverrideFromEnv() throws Exception { diff --git a/dolphinscheduler-common/src/test/resources/remote-logging.yaml b/dolphinscheduler-common/src/test/resources/remote-logging.yaml new file mode 100644 index 0000000000..cb149a77fe --- /dev/null +++ b/dolphinscheduler-common/src/test/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml index 9fc3a3b679..d521e53bc2 100644 --- a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml +++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml @@ -52,6 +52,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml index 70622942f0..5ac9b6350d 100644 --- a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml +++ b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf From 99d8276be7ff1f0cb47f6d67d7440a9380614073 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Wed, 8 May 2024 11:46:09 +0800 Subject: [PATCH 82/88] Optimizing the scope of RPC base classes (#15946) * Optimizing the scope of RPC base classes * Fix UT --- .../alert/rpc/AlertRpcServer.java | 16 +-- .../alert/rpc/AlertRpcServerTest.java | 28 ++--- .../api/service/LoggerServiceTest.java | 19 ++- ....java => AbstractClientMethodInvoker.java} | 5 +- .../base/client/ClientInvocationHandler.java | 5 +- .../base/client/ClientMethodInvoker.java | 2 +- .../base/client/IRpcClientProxyFactory.java | 2 +- .../JdkDynamicRpcClientProxyFactory.java | 5 +- .../base/{ => client}/NettyClientHandler.java | 18 +-- .../{ => client}/NettyRemotingClient.java | 110 ++---------------- .../NettyRemotingClientFactory.java | 2 +- ...gletonJdkDynamicRpcClientProxyFactory.java | 1 - .../base/client/SyncClientMethodInvoker.java | 5 +- .../extract/base/future/ResponseFuture.java | 76 +----------- .../base/server/JdkDynamicServerHandler.java | 12 +- .../{ => server}/NettyRemotingServer.java | 57 +++++---- .../NettyRemotingServerFactory.java | 6 +- .../extract/base/server/RpcServer.java | 74 ++++++++++++ .../base/server/ServerMethodInvoker.java | 4 +- .../base/server/ServerMethodInvokerImpl.java | 9 +- .../server/ServerMethodInvokerRegistry.java | 28 +++++ .../SpringServerMethodInvokerDiscovery.java | 37 ++---- ...onJdkDynamicRpcClientProxyFactoryTest.java | 12 +- .../server/master/rpc/MasterRpcServer.java | 18 +-- .../master/rpc/MasterRpcServerTest.java | 38 ++++++ .../microbench/rpc/RpcBenchMarkTest.java | 14 +-- .../server/worker/rpc/WorkerRpcServer.java | 18 +-- .../worker/rpc/WorkerRpcServerTest.java | 39 +++++++ 28 files changed, 301 insertions(+), 359 deletions(-) rename dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java => dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java (61%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/{BaseRemoteMethodInvoker.java => AbstractClientMethodInvoker.java} (83%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/{ => client}/NettyClientHandler.java (87%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/{ => client}/NettyRemotingClient.java (62%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/{ => client}/NettyRemotingClientFactory.java (95%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/{ => server}/NettyRemotingServer.java (75%) rename dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/{ => server}/NettyRemotingServerFactory.java (84%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java create mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java create mode 100644 dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java index 3bd368573a..d73e4755dd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.alert.rpc; import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; @@ -31,20 +30,7 @@ import org.springframework.stereotype.Service; public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { public AlertRpcServer(AlertConfig alertConfig) { - super(NettyRemotingServerFactory.buildNettyRemotingServer( - NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build())); + super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build()); } - public void start() { - log.info("Starting AlertRpcServer..."); - nettyRemotingServer.start(); - log.info("Started AlertRpcServer..."); - } - - @Override - public void close() { - log.info("Closing AlertRpcServer..."); - nettyRemotingServer.close(); - log.info("Closed AlertRpcServer..."); - } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java similarity index 61% rename from dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java index df88de34e3..75f16848fd 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java @@ -15,22 +15,24 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.alert.rpc; -import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; +import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.mockito.Mockito; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; +import org.junit.jupiter.api.Test; -/** - * dependency config - */ -@Configuration -public class TaskResponseProcessorTestConfig { +class AlertRpcServerTest { - @Bean - public DataQualityResultOperator dataQualityResultOperator() { - return Mockito.mock(DataQualityResultOperator.class); + private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig()); + + @Test + void testStart() { + alertRpcServer.start(); } + + @Test + void testClose() { + alertRpcServer.close(); + } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 4861e1004e..972092602f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -40,7 +40,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.extract.common.ILogService; @@ -91,7 +90,7 @@ public class LoggerServiceTest { @Mock private TaskDefinitionMapper taskDefinitionMapper; - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private int nettyServerPort = 18080; @@ -103,11 +102,10 @@ public class LoggerServiceTest { return; } - nettyRemotingServer = new NettyRemotingServer(NettyServerConfig.builder().listenPort(nettyServerPort).build()); - nettyRemotingServer.start(); - SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery = - new SpringServerMethodInvokerDiscovery(nettyRemotingServer); - springServerMethodInvokerDiscovery.postProcessAfterInitialization(new ILogService() { + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( + NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build()); + springServerMethodInvokerDiscovery.start(); + springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { @Override public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { @@ -142,13 +140,14 @@ public class LoggerServiceTest { public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { } - }, "iLogServiceImpl"); + }); + springServerMethodInvokerDiscovery.start(); } @AfterEach public void tearDown() { - if (nettyRemotingServer != null) { - nettyRemotingServer.close(); + if (springServerMethodInvokerDiscovery != null) { + springServerMethodInvokerDiscovery.close(); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java similarity index 83% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java index 519dd87199..b753f1efa7 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Method; -public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker { +abstract class AbstractClientMethodInvoker implements ClientMethodInvoker { protected final String methodIdentifier; @@ -32,7 +31,7 @@ public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker { protected final Host serverHost; - public BaseRemoteMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + AbstractClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { this.serverHost = serverHost; this.localMethod = localMethod; this.nettyRemotingClient = nettyRemotingClient; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java index d5c9ab73d3..41ec3e056d 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.extract.base.client; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.utils.Host; @@ -31,7 +30,7 @@ import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; @Slf4j -public class ClientInvocationHandler implements InvocationHandler { +class ClientInvocationHandler implements InvocationHandler { private final NettyRemotingClient nettyRemotingClient; @@ -39,7 +38,7 @@ public class ClientInvocationHandler implements InvocationHandler { private final Host serverHost; - public ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) { + ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) { this.serverHost = checkNotNull(serverHost); this.nettyRemotingClient = checkNotNull(nettyRemotingClient); this.methodInvokerMap = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java index dcf53b0311..a287fd95ce 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.extract.base.client; import java.lang.reflect.Method; -public interface ClientMethodInvoker { +interface ClientMethodInvoker { Object invoke(Object proxy, Method method, Object[] args) throws Throwable; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java index e60b0f18b0..afd3adf348 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.extract.base.client; -public interface IRpcClientProxyFactory { +interface IRpcClientProxyFactory { /** * Create the client proxy. diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java index 5635a88f34..bf329ab3fc 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Proxy; @@ -34,7 +33,7 @@ import com.google.common.cache.LoadingCache; /** * This class is used to create a proxy client which will transform local method invocation to remove invocation. */ -public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { +class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { private final NettyRemotingClient nettyRemotingClient; @@ -49,7 +48,7 @@ public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { } }); - public JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) { + JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) { this.nettyRemotingClient = nettyRemotingClient; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java similarity index 87% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java index b0d998af83..be570eb577 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java @@ -15,16 +15,15 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; +import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; import org.apache.dolphinscheduler.extract.base.protocal.Transporter; import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; -import java.util.concurrent.ExecutorService; - import lombok.extern.slf4j.Slf4j; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; @@ -38,11 +37,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private final NettyRemotingClient nettyRemotingClient; - private final ExecutorService callbackExecutor; - - public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) { + public NettyClientHandler(NettyRemotingClient nettyRemotingClient) { this.nettyRemotingClient = nettyRemotingClient; - this.callbackExecutor = callbackExecutor; } @Override @@ -64,13 +60,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { } StandardRpcResponse deserialize = JsonSerializer.deserialize(transporter.getBody(), StandardRpcResponse.class); future.setIRpcResponse(deserialize); - future.release(); - if (future.getInvokeCallback() != null) { - future.removeFuture(); - this.callbackExecutor.execute(future::executeInvokeCallback); - } else { - future.putResponse(deserialize); - } + future.putResponse(deserialize); } @Override diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java similarity index 62% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index e4682f5224..3999f5c9f5 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -15,33 +15,24 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; -import org.apache.dolphinscheduler.extract.base.exception.RemotingTooMuchRequestException; -import org.apache.dolphinscheduler.extract.base.future.InvokeCallback; -import org.apache.dolphinscheduler.extract.base.future.ReleaseSemaphore; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; import org.apache.dolphinscheduler.extract.base.protocal.Transporter; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; -import org.apache.dolphinscheduler.extract.base.utils.CallerThreadExecutePolicy; import org.apache.dolphinscheduler.extract.base.utils.Constants; import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -71,14 +62,8 @@ public class NettyRemotingClient implements AutoCloseable { private final NettyClientConfig clientConfig; - private final Semaphore asyncSemaphore = new Semaphore(1024, true); - - private final ExecutorService callbackExecutor; - private final NettyClientHandler clientHandler; - private final ScheduledExecutorService responseFutureExecutor; - public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-"); @@ -87,18 +72,7 @@ public class NettyRemotingClient implements AutoCloseable { } else { this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory); } - this.callbackExecutor = new ThreadPoolExecutor( - Constants.CPUS, - Constants.CPUS, - 1, - TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), - ThreadUtils.newDaemonThreadFactory("NettyClientCallbackThread-"), - new CallerThreadExecutePolicy()); - this.clientHandler = new NettyClientHandler(this, callbackExecutor); - - this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor( - ThreadUtils.newDaemonThreadFactory("NettyClientResponseFutureThread-")); + this.clientHandler = new NettyClientHandler(this); this.start(); } @@ -127,66 +101,9 @@ public class NettyRemotingClient implements AutoCloseable { .addLast(new TransporterDecoder(), clientHandler, new TransporterEncoder()); } }); - this.responseFutureExecutor.scheduleWithFixedDelay(ResponseFuture::scanFutureTable, 0, 1, TimeUnit.SECONDS); isStarted.compareAndSet(false, true); } - public void sendAsync(final Host host, - final Transporter transporter, - final long timeoutMillis, - final InvokeCallback invokeCallback) throws InterruptedException, RemotingException { - final Channel channel = getChannel(host); - if (channel == null) { - throw new RemotingException("network error"); - } - /* - * request unique identification - */ - final long opaque = transporter.getHeader().getOpaque(); - /* - * control concurrency number - */ - boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); - if (acquired) { - final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore); - - /* - * response future - */ - final ResponseFuture responseFuture = new ResponseFuture(opaque, - timeoutMillis, - invokeCallback, - releaseSemaphore); - try { - channel.writeAndFlush(transporter).addListener(future -> { - if (future.isSuccess()) { - responseFuture.setSendOk(true); - return; - } else { - responseFuture.setSendOk(false); - } - responseFuture.setCause(future.cause()); - responseFuture.putResponse(null); - try { - responseFuture.executeInvokeCallback(); - } catch (Exception ex) { - log.error("execute callback error", ex); - } finally { - responseFuture.release(); - } - }); - } catch (Exception ex) { - responseFuture.release(); - throw new RemotingException(String.format("Send transporter to host: %s failed", host), ex); - } - } else { - String message = String.format( - "try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", - timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); - throw new RemotingTooMuchRequestException(message); - } - } - public IRpcResponse sendSync(final Host host, final Transporter transporter, final long timeoutMillis) throws InterruptedException, RemotingException { final Channel channel = getChannel(host); @@ -194,7 +111,7 @@ public class NettyRemotingClient implements AutoCloseable { throw new RemotingException(String.format("connect to : %s fail", host)); } final long opaque = transporter.getHeader().getOpaque(); - final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); + final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis); channel.writeAndFlush(transporter).addListener(future -> { if (future.isSuccess()) { responseFuture.setSendOk(true); @@ -220,7 +137,7 @@ public class NettyRemotingClient implements AutoCloseable { return iRpcResponse; } - public Channel getChannel(Host host) { + private Channel getChannel(Host host) { Channel channel = channels.get(host); if (channel != null && channel.isActive()) { return channel; @@ -235,9 +152,9 @@ public class NettyRemotingClient implements AutoCloseable { * @param isSync sync flag * @return channel */ - public Channel createChannel(Host host, boolean isSync) { - ChannelFuture future; + private Channel createChannel(Host host, boolean isSync) { try { + ChannelFuture future; synchronized (bootstrap) { future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); } @@ -249,10 +166,11 @@ public class NettyRemotingClient implements AutoCloseable { channels.put(host, channel); return channel; } - } catch (Exception ex) { - log.warn(String.format("connect to %s error", host), ex); + throw new IllegalArgumentException("connect to host: " + host + " failed"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Connect to host: " + host + " failed", e); } - return null; } @Override @@ -263,12 +181,6 @@ public class NettyRemotingClient implements AutoCloseable { if (workerGroup != null) { this.workerGroup.shutdownGracefully(); } - if (callbackExecutor != null) { - this.callbackExecutor.shutdownNow(); - } - if (this.responseFutureExecutor != null) { - this.responseFutureExecutor.shutdownNow(); - } log.info("netty client closed"); } catch (Exception ex) { log.error("netty client close exception", ex); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java similarity index 95% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java index 7bbebfbf3d..d14a8aa54e 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java index 28d82532be..44d310e70b 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClientFactory; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; public class SingletonJdkDynamicRpcClientProxyFactory { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java index b5fdf3fb71..4731a22d0a 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.extract.base.IRpcResponse; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException; @@ -29,9 +28,9 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Method; -public class SyncClientMethodInvoker extends BaseRemoteMethodInvoker { +class SyncClientMethodInvoker extends AbstractClientMethodInvoker { - public SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { super(serverHost, localMethod, nettyRemotingClient); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java index 35405c5578..1fbbd9ed6c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.extract.base.future; import org.apache.dolphinscheduler.extract.base.IRpcResponse; -import java.util.Iterator; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -34,17 +32,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class ResponseFuture { - private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); + private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(); private final long opaque; // remove the timeout private final long timeoutMillis; - private final InvokeCallback invokeCallback; - - private final ReleaseSemaphore releaseSemaphore; - private final CountDownLatch latch = new CountDownLatch(1); private final long beginTimestamp = System.currentTimeMillis(); @@ -57,14 +51,9 @@ public class ResponseFuture { private Throwable cause; - public ResponseFuture(long opaque, - long timeoutMillis, - InvokeCallback invokeCallback, - ReleaseSemaphore releaseSemaphore) { + public ResponseFuture(long opaque, long timeoutMillis) { this.opaque = opaque; this.timeoutMillis = timeoutMillis; - this.invokeCallback = invokeCallback; - this.releaseSemaphore = releaseSemaphore; FUTURE_TABLE.put(opaque, this); } @@ -90,10 +79,6 @@ public class ResponseFuture { return FUTURE_TABLE.get(opaque); } - public void removeFuture() { - FUTURE_TABLE.remove(opaque); - } - /** * whether timeout * @@ -104,15 +89,6 @@ public class ResponseFuture { return diff > this.timeoutMillis; } - /** - * execute invoke callback - */ - public void executeInvokeCallback() { - if (invokeCallback != null) { - invokeCallback.operationComplete(this); - } - } - public boolean isSendOK() { return sendOk; } @@ -129,52 +105,4 @@ public class ResponseFuture { return cause; } - public long getOpaque() { - return opaque; - } - - public long getTimeoutMillis() { - return timeoutMillis; - } - - public long getBeginTimestamp() { - return beginTimestamp; - } - - public InvokeCallback getInvokeCallback() { - return invokeCallback; - } - - /** - * release - */ - public void release() { - if (this.releaseSemaphore != null) { - this.releaseSemaphore.release(); - } - } - - /** - * scan future table - */ - public static void scanFutureTable() { - Iterator> it = FUTURE_TABLE.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry next = it.next(); - ResponseFuture future = next.getValue(); - if ((future.getBeginTimestamp() + future.getTimeoutMillis() + 1000) > System.currentTimeMillis()) { - continue; - } - try { - // todo: use thread pool to execute the async callback, otherwise will block the scan thread - future.release(); - future.executeInvokeCallback(); - } catch (Exception ex) { - log.error("ScanFutureTable, execute callback error, requestId: {}", future.getOpaque(), ex); - } - it.remove(); - log.debug("Remove timeout request: {}", future); - } - } - } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java index b4978172f1..f57ff0b609 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.extract.base.server; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; @@ -30,6 +29,7 @@ import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import lombok.extern.slf4j.Slf4j; @@ -42,14 +42,14 @@ import io.netty.handler.timeout.IdleStateEvent; @Slf4j @ChannelHandler.Sharable -public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { +class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { - private final NettyRemotingServer nettyRemotingServer; + private final ExecutorService methodInvokeExecutor; private final Map methodInvokerMap; - public JdkDynamicServerHandler(NettyRemotingServer nettyRemotingServer) { - this.nettyRemotingServer = nettyRemotingServer; + JdkDynamicServerHandler(ExecutorService methodInvokeExecutor) { + this.methodInvokeExecutor = methodInvokeExecutor; this.methodInvokerMap = new ConcurrentHashMap<>(); } @@ -90,7 +90,7 @@ public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { channel.writeAndFlush(response); return; } - nettyRemotingServer.getDefaultExecutor().execute(() -> { + methodInvokeExecutor.execute(() -> { StandardRpcResponse iRpcResponse; try { StandardRpcRequest standardRpcRequest = diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java similarity index 75% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index 365a17dd03..9beeaced3d 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -15,15 +15,13 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.exception.RemoteException; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; -import org.apache.dolphinscheduler.extract.base.server.JdkDynamicServerHandler; -import org.apache.dolphinscheduler.extract.base.server.ServerMethodInvoker; import org.apache.dolphinscheduler.extract.base.utils.Constants; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; @@ -32,6 +30,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; @@ -48,12 +47,15 @@ import io.netty.handler.timeout.IdleStateHandler; * remoting netty server */ @Slf4j -public class NettyRemotingServer { +class NettyRemotingServer { private final ServerBootstrap serverBootstrap = new ServerBootstrap(); - private final ExecutorService defaultExecutor = ThreadUtils - .newDaemonFixedThreadExecutor("NettyRemotingServerThread", Runtime.getRuntime().availableProcessors() * 2); + @Getter + private final String serverName; + + @Getter + private final ExecutorService methodInvokerExecutor; private final EventLoopGroup bossGroup; @@ -61,16 +63,20 @@ public class NettyRemotingServer { private final NettyServerConfig serverConfig; - private final JdkDynamicServerHandler serverHandler = new JdkDynamicServerHandler(this); + private final JdkDynamicServerHandler channelHandler; private final AtomicBoolean isStarted = new AtomicBoolean(false); - public NettyRemotingServer(final NettyServerConfig serverConfig) { + NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; + this.serverName = serverConfig.getServerName(); + this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor( + serverName + "MethodInvoker-%d", Runtime.getRuntime().availableProcessors() * 2 + 1); + this.channelHandler = new JdkDynamicServerHandler(methodInvokerExecutor); ThreadFactory bossThreadFactory = - ThreadUtils.newDaemonThreadFactory(serverConfig.getServerName() + "BossThread_%s"); + ThreadUtils.newDaemonThreadFactory(serverName + "BossThread-%d"); ThreadFactory workerThreadFactory = - ThreadUtils.newDaemonThreadFactory(serverConfig.getServerName() + "WorkerThread_%s"); + ThreadUtils.newDaemonThreadFactory(serverName + "WorkerThread-%d"); if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory); @@ -80,7 +86,7 @@ public class NettyRemotingServer { } } - public void start() { + void start() { if (isStarted.compareAndSet(false, true)) { this.serverBootstrap .group(this.bossGroup, this.workGroup) @@ -103,9 +109,9 @@ public class NettyRemotingServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - log.error("{} bind fail {}, exit", serverConfig.getServerName(), e.getMessage(), e); throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort())); + String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), + e); } if (future.isSuccess()) { @@ -113,14 +119,9 @@ public class NettyRemotingServer { return; } - if (future.cause() != null) { - throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), - future.cause()); - } else { - throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort())); - } + throw new RemoteException( + String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), + future.cause()); } } @@ -135,18 +136,14 @@ public class NettyRemotingServer { .addLast("decoder", new TransporterDecoder()) .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) - .addLast("handler", serverHandler); + .addLast("handler", channelHandler); } - public ExecutorService getDefaultExecutor() { - return defaultExecutor; + void registerMethodInvoker(ServerMethodInvoker methodInvoker) { + channelHandler.registerMethodInvoker(methodInvoker); } - public void registerMethodInvoker(ServerMethodInvoker methodInvoker) { - serverHandler.registerMethodInvoker(methodInvoker); - } - - public void close() { + void close() { if (isStarted.compareAndSet(true, false)) { try { if (bossGroup != null) { @@ -155,7 +152,7 @@ public class NettyRemotingServer { if (workGroup != null) { this.workGroup.shutdownGracefully(); } - defaultExecutor.shutdown(); + methodInvokerExecutor.shutdown(); } catch (Exception ex) { log.error("netty server close exception", ex); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java similarity index 84% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java index 6bf1b8d31c..70ed0529e8 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import lombok.experimental.UtilityClass; @UtilityClass -public class NettyRemotingServerFactory { +class NettyRemotingServerFactory { - public NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { + NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { return new NettyRemotingServer(nettyServerConfig); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java new file mode 100644 index 0000000000..213868ba46 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; + +import java.lang.reflect.Method; + +import lombok.extern.slf4j.Slf4j; + +/** + * The RpcServer based on Netty. The server will register the method invoker and provide the service to the client. + * Once the server is started, it will listen on the port and wait for the client to connect. + * 
+ *          RpcServer rpcServer = new RpcServer(new NettyServerConfig());
+ *          rpcServer.registerServerMethodInvokerProvider(new ServerMethodInvokerProviderImpl());
+ *          rpcServer.start();
+ *
+ */ +@Slf4j +public class RpcServer implements ServerMethodInvokerRegistry, AutoCloseable { + + private final NettyRemotingServer nettyRemotingServer; + + public RpcServer(NettyServerConfig nettyServerConfig) { + this.nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig); + } + + public void start() { + nettyRemotingServer.start(); + } + + @Override + public void registerServerMethodInvokerProvider(Object serverMethodInvokerProviderBean) { + for (Class anInterface : serverMethodInvokerProviderBean.getClass().getInterfaces()) { + if (anInterface.getAnnotation(RpcService.class) == null) { + continue; + } + for (Method method : anInterface.getDeclaredMethods()) { + RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class); + if (rpcMethod == null) { + continue; + } + ServerMethodInvoker serverMethodInvoker = + new ServerMethodInvokerImpl(serverMethodInvokerProviderBean, method); + nettyRemotingServer.registerMethodInvoker(serverMethodInvoker); + log.debug("Register ServerMethodInvoker: {} to bean: {}", + serverMethodInvoker.getMethodIdentify(), serverMethodInvoker.getMethodProviderIdentify()); + } + } + } + + @Override + public void close() { + nettyRemotingServer.close(); + } +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java index ee633217b2..151b54bb97 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java @@ -17,10 +17,12 @@ package org.apache.dolphinscheduler.extract.base.server; -public interface ServerMethodInvoker { +interface ServerMethodInvoker { String getMethodIdentify(); + String getMethodProviderIdentify(); + Object invoke(final Object... arg) throws Throwable; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java index eea9da5e14..4c29650aa0 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.extract.base.server; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -public class ServerMethodInvokerImpl implements ServerMethodInvoker { +class ServerMethodInvokerImpl implements ServerMethodInvoker { private final Object serviceBean; @@ -28,7 +28,7 @@ public class ServerMethodInvokerImpl implements ServerMethodInvoker { private final String methodIdentify; - public ServerMethodInvokerImpl(Object serviceBean, Method method) { + ServerMethodInvokerImpl(Object serviceBean, Method method) { this.serviceBean = serviceBean; this.method = method; this.methodIdentify = method.toGenericString(); @@ -48,4 +48,9 @@ public class ServerMethodInvokerImpl implements ServerMethodInvoker { public String getMethodIdentify() { return methodIdentify; } + + @Override + public String getMethodProviderIdentify() { + return serviceBean.getClass().getName(); + } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java new file mode 100644 index 0000000000..4e56be2617 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +interface ServerMethodInvokerRegistry { + + /** + * Register service object, which will be used to invoke the {@link ServerMethodInvoker}. + * The serverMethodInvokerProviderObject should implement with interface which contains {@link org.apache.dolphinscheduler.extract.base.RpcService} annotation. + */ + void registerServerMethodInvokerProvider(Object serverMethodInvokerProviderObject); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java index 2b87a70080..de4943990c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java @@ -17,11 +17,7 @@ package org.apache.dolphinscheduler.extract.base.server; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; -import org.apache.dolphinscheduler.extract.base.RpcMethod; -import org.apache.dolphinscheduler.extract.base.RpcService; - -import java.lang.reflect.Method; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import lombok.extern.slf4j.Slf4j; @@ -29,38 +25,21 @@ import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; import org.springframework.lang.Nullable; +/** + * The RpcServer which will auto discovery the {@link ServerMethodInvoker} from Spring container. + */ @Slf4j -public class SpringServerMethodInvokerDiscovery implements BeanPostProcessor { +public class SpringServerMethodInvokerDiscovery extends RpcServer implements BeanPostProcessor { - protected final NettyRemotingServer nettyRemotingServer; - - public SpringServerMethodInvokerDiscovery(NettyRemotingServer nettyRemotingServer) { - this.nettyRemotingServer = nettyRemotingServer; + public SpringServerMethodInvokerDiscovery(NettyServerConfig nettyServerConfig) { + super(nettyServerConfig); } @Nullable @Override public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - Class[] interfaces = bean.getClass().getInterfaces(); - for (Class anInterface : interfaces) { - if (anInterface.getAnnotation(RpcService.class) == null) { - continue; - } - registerRpcMethodInvoker(anInterface, bean, beanName); - } + registerServerMethodInvokerProvider(bean); return bean; } - private void registerRpcMethodInvoker(Class anInterface, Object bean, String beanName) { - Method[] declaredMethods = anInterface.getDeclaredMethods(); - for (Method method : declaredMethods) { - RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class); - if (rpcMethod == null) { - continue; - } - ServerMethodInvoker methodInvoker = new ServerMethodInvokerImpl(bean, method); - nettyRemotingServer.registerMethodInvoker(methodInvoker); - log.debug("Register ServerMethodInvoker: {} to bean: {}", methodInvoker.getMethodIdentify(), beanName); - } - } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java index 521cf7c75a..92ed49934c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.extract.base.client; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.RpcService; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; @@ -37,7 +36,7 @@ import org.junit.jupiter.api.Test; public class SingletonJdkDynamicRpcClientProxyFactoryTest { - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private String serverAddress; @@ -48,11 +47,10 @@ public class SingletonJdkDynamicRpcClientProxyFactoryTest { .serverName("ApiServer") .listenPort(listenPort) .build(); - nettyRemotingServer = new NettyRemotingServer(nettyServerConfig); - nettyRemotingServer.start(); serverAddress = "localhost:" + listenPort; - new SpringServerMethodInvokerDiscovery(nettyRemotingServer) - .postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); + springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new IServiceImpl()); + springServerMethodInvokerDiscovery.start(); } @Test @@ -82,7 +80,7 @@ public class SingletonJdkDynamicRpcClientProxyFactoryTest { @AfterEach public void tearDown() { - nettyRemotingServer.close(); + springServerMethodInvokerDiscovery.close(); } @RpcService diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java index 0eaf885d11..ab89b021d6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.rpc; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -31,21 +30,8 @@ import org.springframework.stereotype.Component; public class MasterRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { public MasterRpcServer(MasterConfig masterConfig) { - super(NettyRemotingServerFactory.buildNettyRemotingServer(NettyServerConfig.builder() - .serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()).build())); - } - - public void start() { - log.info("Starting MasterRPCServer..."); - nettyRemotingServer.start(); - log.info("Started MasterRPCServer..."); - } - - @Override - public void close() { - log.info("Closing MasterRPCServer..."); - nettyRemotingServer.close(); - log.info("Closed MasterRPCServer..."); + super(NettyServerConfig.builder().serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()) + .build()); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java new file mode 100644 index 0000000000..1e5a77edb3 --- /dev/null +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.server.master.config.MasterConfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MasterRpcServerTest { + + private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig()); + + @Test + void testStart() { + Assertions.assertDoesNotThrow(masterRpcServer::start); + } + + @Test + void testClose() { + Assertions.assertDoesNotThrow(masterRpcServer::close); + } +} diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java index 1a3e4ab1e2..496983118f 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.microbench.rpc; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; @@ -46,18 +45,17 @@ import org.openjdk.jmh.infra.Blackhole; @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) public class RpcBenchMarkTest extends AbstractBaseBenchmark { - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private IService iService; @Setup public void before() { - nettyRemotingServer = new NettyRemotingServer( - NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build()); - nettyRemotingServer.start(); - SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery = - new SpringServerMethodInvokerDiscovery(nettyRemotingServer); + NettyServerConfig nettyServerConfig = + NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build(); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); springServerMethodInvokerDiscovery.postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); + springServerMethodInvokerDiscovery.start(); iService = SingletonJdkDynamicRpcClientProxyFactory.getProxyClient("localhost:12345", IService.class); } @@ -72,6 +70,6 @@ public class RpcBenchMarkTest extends AbstractBaseBenchmark { @TearDown public void after() { - nettyRemotingServer.close(); + springServerMethodInvokerDiscovery.close(); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java index 7733fbba4f..b9f3855cf9 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.rpc; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -33,21 +32,8 @@ import org.springframework.stereotype.Service; public class WorkerRpcServer extends SpringServerMethodInvokerDiscovery implements Closeable { public WorkerRpcServer(WorkerConfig workerConfig) { - super(NettyRemotingServerFactory.buildNettyRemotingServer(NettyServerConfig.builder() - .serverName("WorkerRpcServer").listenPort(workerConfig.getListenPort()).build())); - } - - public void start() { - log.info("WorkerRpcServer starting..."); - nettyRemotingServer.start(); - log.info("WorkerRpcServer started..."); - } - - @Override - public void close() { - log.info("WorkerRpcServer closing"); - nettyRemotingServer.close(); - log.info("WorkerRpcServer closed"); + super(NettyServerConfig.builder().serverName("WorkerRpcServer").listenPort(workerConfig.getListenPort()) + .build()); } } diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java new file mode 100644 index 0000000000..d27eaeeadf --- /dev/null +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.rpc; + +import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class WorkerRpcServerTest { + + private final WorkerRpcServer workerRpcServer = new WorkerRpcServer(new WorkerConfig()); + + @Test + void testStart() { + Assertions.assertDoesNotThrow(workerRpcServer::start); + } + + @Test + void testClose() { + Assertions.assertDoesNotThrow(workerRpcServer::close); + } + +} From 8426d2346cde5431ad83a23cbce20864026c556a Mon Sep 17 00:00:00 2001 From: xiangzihao <460888207@qq.com> Date: Thu, 9 May 2024 11:01:05 +0800 Subject: [PATCH 83/88] [HotFix] [CI] Temporary skipping mergeable check (#15958) * temporary skipping mergeable check --- .asf.yaml | 2 +- .github/mergeable.yml | 62 -------------------------- .github/workflows/mergeable.yml | 78 +++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 63 deletions(-) delete mode 100644 .github/mergeable.yml create mode 100644 .github/workflows/mergeable.yml diff --git a/.asf.yaml b/.asf.yaml index 84619447be..abeb08c0b8 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -46,7 +46,7 @@ github: - E2E - Docs - Frontend Build - - "Mergeable: milestone-label-check" +# - "Mergeable: milestone-label-check" required_pull_request_reviews: dismiss_stale_reviews: true required_approving_review_count: 2 diff --git a/.github/mergeable.yml b/.github/mergeable.yml deleted file mode 100644 index a1df2e7410..0000000000 --- a/.github/mergeable.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -version: 2 -mergeable: - # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643, - # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as - # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like, - # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened - - when: pull_request.opened, pull_request.reopened, pull_request.synchronize - name: sync-sql-ddl - validate: - # Sql files must change synchronize - - do: dependent - files: - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql' - message: 'Sql files not change synchronize' - # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize - fail: - - do: comment - payload: - body: > - :warning: This PR do not change database DDL synchronize. - leave_old_comment: false - - do: labels - add: 'sql not sync' - # Remove labels 'sql not sync' if pass - pass: - - do: labels - delete: 'sql not sync' - - - when: pull_request.* - name: milestone-label-check - validate: - - do: milestone - no_empty: - enabled: false # Cannot be empty when true. - message: 'Milestone is required and cannot be empty.' - - do: label - and: - - must_include: - regex: 'feature|bug|improvement|document|chore|revert' - message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`' - - must_include: - regex: 'ready-to-merge' - message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.' diff --git a/.github/workflows/mergeable.yml b/.github/workflows/mergeable.yml new file mode 100644 index 0000000000..8b7bd8799c --- /dev/null +++ b/.github/workflows/mergeable.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +#version: 2 + +on: + pull_request: + +name: "Mergeable" + +jobs: + result: + name: "Mergeable: milestone-label-check" + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Status + run: | + echo "Temporary skipping this check" + +#mergeable: +# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643, +# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as +# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like, +# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened +# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize +# name: sync-sql-ddl +# validate: +# # Sql files must change synchronize +# - do: dependent +# files: +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql' +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql' +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql' +# message: 'Sql files not change synchronize' +# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize +# fail: +# - do: comment +# payload: +# body: > +# :warning: This PR do not change database DDL synchronize. +# leave_old_comment: false +# - do: labels +# add: 'sql not sync' +# # Remove labels 'sql not sync' if pass +# pass: +# - do: labels +# delete: 'sql not sync' +# +# - when: pull_request.* +# name: milestone-label-check +# validate: +# - do: milestone +# no_empty: +# enabled: false # Cannot be empty when true. +# message: 'Milestone is required and cannot be empty.' +# - do: label +# and: +# - must_include: +# regex: 'feature|bug|improvement|document|chore|revert' +# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`' +# - must_include: +# regex: 'ready-to-merge' +# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.' From bbca37d03eb255299a0527f299aa90b96e0e1218 Mon Sep 17 00:00:00 2001 From: privking <43061765+privking@users.noreply.github.com> Date: Thu, 9 May 2024 11:36:56 +0800 Subject: [PATCH 84/88] [FIX] Completed tasks cannot be re-executed in a workflow instance (#15884) * fix bug: Failed to resume stopped workflow instance * Revert "fix bug: Failed to resume stopped workflow instance" This reverts commit 1546e9d5a51178a94bedd18a718b15431355428b. * fix bug : Completed tasks cannot be re-executed in a workflow instance --------- Co-authored-by: Rick Cheng --- .../server/master/runner/WorkflowExecuteRunnable.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index eafba17f69..725b7e000a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -2120,13 +2120,9 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { workflowInstance.setVarPool(JSONUtils.toJsonString(processProperties)); processInstanceDao.updateById(workflowInstance); - // remove task instance from taskInstanceMap, completeTaskSet, validTaskMap, errorTaskMap - // completeTaskSet remove dependency taskInstanceMap, so the sort can't change - completeTaskSet.removeIf(taskCode -> { - Optional existTaskInstanceOptional = getTaskInstance(taskCode); - return existTaskInstanceOptional - .filter(taskInstance -> dag.containsNode(taskInstance.getTaskCode())).isPresent(); - }); + // remove task instance from taskInstanceMap,taskCodeInstanceMap , completeTaskSet, validTaskMap, errorTaskMap + completeTaskSet.removeIf(dag::containsNode); + taskCodeInstanceMap.entrySet().removeIf(entity -> dag.containsNode(entity.getValue().getTaskCode())); taskInstanceMap.entrySet().removeIf(entry -> dag.containsNode(entry.getValue().getTaskCode())); validTaskMap.entrySet().removeIf(entry -> dag.containsNode(entry.getKey())); errorTaskMap.entrySet().removeIf(entry -> dag.containsNode(entry.getKey())); From 8d336def6140d01bf2d6007014a9689b237baab7 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 9 May 2024 12:23:01 +0800 Subject: [PATCH 85/88] [DSIP-35][Alert] Refactor the alert thread model (#15932) --- .../plugin/alert/voice/VoiceAlertChannel.java | 2 +- .../plugin/alert/voice/VoiceSender.java | 4 +- .../src/test/java/VoiceSenderTest.java | 2 +- .../alert/api/AlertChannel.java | 2 +- .../dolphinscheduler/alert/api/AlertData.java | 8 - .../alert/api/AlertResult.java | 14 +- .../alert/dingtalk/DingTalkAlertChannel.java | 2 +- .../plugin/alert/dingtalk/DingTalkSender.java | 6 +- .../alert/dingtalk/DingTalkSenderTest.java | 2 +- .../plugin/alert/email/EmailAlertChannel.java | 10 +- .../plugin/alert/email/MailSender.java | 6 +- .../alert/email/EmailAlertChannelTest.java | 2 +- .../plugin/alert/email/MailUtilsTest.java | 14 +- .../alert/feishu/FeiShuAlertChannel.java | 2 +- .../plugin/alert/feishu/FeiShuSender.java | 6 +- .../plugin/alert/feishu/FeiShuSenderTest.java | 6 +- .../plugin/alert/http/HttpAlertChannel.java | 2 +- .../plugin/alert/http/HttpSender.java | 6 +- .../alert/http/HttpAlertChannelTest.java | 4 +- .../plugin/alert/http/HttpSenderTest.java | 2 +- .../pagerduty/PagerDutyAlertChannel.java | 4 +- .../alert/pagerduty/PagerDutySender.java | 4 +- .../alert/pagerduty/PagerDutySenderTest.java | 2 +- .../prometheus/PrometheusAlertChannel.java | 2 +- .../prometheus/PrometheusAlertSender.java | 12 +- .../prometheus/PrometheusAlertSenderTest.java | 6 +- .../alert/script/ScriptAlertChannel.java | 2 +- .../plugin/alert/script/ScriptSender.java | 6 +- .../plugin/alert/script/ScriptSenderTest.java | 16 +- .../plugin/alert/slack/SlackAlertChannel.java | 6 +- .../alert/telegram/TelegramAlertChannel.java | 2 +- .../plugin/alert/telegram/TelegramSender.java | 6 +- .../alert/telegram/TelegramSenderTest.java | 10 +- .../webexteams/WebexTeamsAlertChannel.java | 2 +- .../alert/webexteams/WebexTeamsSender.java | 4 +- .../webexteams/WebexTeamsSenderTest.java | 2 +- .../alert/wechat/WeChatAlertChannel.java | 2 +- .../plugin/alert/wechat/WeChatSender.java | 11 +- .../plugin/alert/wechat/WeChatSenderTest.java | 4 +- .../dolphinscheduler-alert-server/pom.xml | 6 + .../dolphinscheduler/alert/AlertServer.java | 46 +-- .../alert/config/AlertConfig.java | 6 + .../alert/metrics/AlertServerMetrics.java | 6 + .../alert/plugin/AlertPluginManager.java | 2 +- .../alert/registry/AlertHeartbeatTask.java | 8 +- .../alert/registry/AlertRegistryClient.java | 9 +- .../alert/rpc/AlertOperatorImpl.java | 11 +- .../alert/service/AbstractEventFetcher.java | 100 +++++ .../alert/service/AbstractEventLoop.java | 101 +++++ .../service/AbstractEventPendingQueue.java | 53 +++ .../alert/service/AbstractEventSender.java | 191 +++++++++ .../alert/service/AlertBootstrapService.java | 390 +++--------------- .../alert/service/AlertEventFetcher.java | 51 +++ .../alert/service/AlertEventLoop.java | 46 +++ .../alert/service/AlertEventPendingQueue.java | 33 ++ .../alert/service/AlertHAServer.java | 36 ++ .../alert/service/AlertSender.java | 131 ++++++ .../service/AlertSenderThreadPoolFactory.java | 41 ++ .../alert/service/EventFetcher.java | 34 ++ .../alert/service/EventLoop.java | 47 +++ .../alert/service/EventPendingQueue.java | 34 ++ .../alert/service/EventSender.java | 33 ++ .../alert/service/ListenerEventFetcher.java | 51 +++ .../alert/service/ListenerEventLoop.java | 40 ++ .../service/ListenerEventPendingQueue.java | 32 ++ .../service/ListenerEventPostService.java | 262 ------------ .../alert/service/ListenerEventSender.java | 146 +++++++ .../src/main/resources/application.yaml | 3 +- .../alert/config/AlertConfigTest.java | 43 ++ ...pServiceTest.java => AlertSenderTest.java} | 74 +--- ...Test.java => ListenerEventSenderTest.java} | 47 +-- .../service/AlertEventPendingQueueTest.java | 94 +++++ .../AlertSenderThreadPoolFactoryTest.java | 44 ++ .../src/test/resources/application.yaml | 107 +++++ .../common/enums/ServerStatus.java | 2 +- .../common/model/AlertServerHeartBeat.java | 5 + .../apache/dolphinscheduler/dao/AlertDao.java | 35 +- .../dao/mapper/AlertMapper.java | 5 +- .../dao/mapper/ListenerEventMapper.java | 3 +- .../dao/repository/ListenerEventDao.java | 31 ++ .../repository/impl/ListenerEventDaoImpl.java | 51 +++ .../dao/mapper/AlertMapper.xml | 4 +- .../dao/mapper/ListenerEventMapper.xml | 3 +- .../dao/mapper/ListenerEventMapperTest.java | 4 +- .../dao/repository/impl/AlertDaoTest.java | 28 +- .../impl/ListenerEventDaoImplTest.java | 79 ++++ .../alert/request/AlertSendResponse.java | 16 + .../registry/api/Registry.java | 7 +- .../registry/api/ha/AbstractHAServer.java | 105 +++++ .../AbstractServerStatusChangeListener.java | 42 ++ .../ha/DefaultServerStatusChangeListener.java | 34 ++ .../registry/api/ha/HAServer.java | 68 +++ .../api/ha/ServerStatusChangeListener.java | 24 ++ .../plugin/registry/etcd/EtcdRegistry.java | 31 ++ .../etcd/EtcdKeepAliveLeaseManagerTest.java | 6 +- .../plugin/registry/jdbc/JdbcRegistry.java | 11 + .../jdbc/task/RegistryLockManager.java | 24 ++ .../registry/zookeeper/ZookeeperRegistry.java | 50 ++- .../src/main/resources/application.yaml | 3 +- 99 files changed, 2344 insertions(+), 890 deletions(-) create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java delete mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java rename dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/{AlertBootstrapServiceTest.java => AlertSenderTest.java} (73%) rename dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/{ListenerEventPostServiceTest.java => ListenerEventSenderTest.java} (82%) create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java index eeaaba5d01..4aa29c19c5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java @@ -40,7 +40,7 @@ public final class VoiceAlertChannel implements AlertChannel { Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "aliyun-voice params is null"); + return new AlertResult(false, "aliyun-voice params is null"); } VoiceParam voiceParam = buildVoiceParam(paramsMap); return new VoiceSender(voiceParam).send(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java index c6c29d8735..fe0fc65986 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java @@ -46,7 +46,7 @@ public final class VoiceSender { public AlertResult send() { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); try { Client client = createClient(voiceParam.getConnection()); SingleCallByTtsRequest singleCallByTtsRequest = new SingleCallByTtsRequest() @@ -61,7 +61,7 @@ public final class VoiceSender { } SingleCallByTtsResponseBody body = response.getBody(); if (body.code.equalsIgnoreCase("ok")) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage(body.getCallId()); } else { alertResult.setMessage(body.getMessage()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java index 515a410b63..15f4871392 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java @@ -46,7 +46,7 @@ class VoiceSenderTest { VoiceSender weChatSender = new VoiceSender(voiceParam); AlertResult alertResult = weChatSender.send(); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java index 530a548342..a4eaae232f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java @@ -35,6 +35,6 @@ public interface AlertChannel { AlertResult process(AlertInfo info); default @NonNull AlertResult closeAlert(AlertInfo info) { - return new AlertResult("true", "no need to close alert"); + return new AlertResult(true, "no need to close alert"); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java index 37a3f3357c..004e8b3bda 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java @@ -53,14 +53,6 @@ public class AlertData { */ private String log; - /** - * 0 do not send warning; - * 1 send if process success; - * 2 send if process failed; - * 3 send if process ends, whatever the result; - */ - private int warnType; - /** * AlertType#code */ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java index b6c5db38e9..ceeed97510 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java @@ -33,15 +33,19 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class AlertResult { - /** - * todo: use enum - * false or true - */ - private String status; + private boolean success; /** * alert result message, each plugin can have its own message */ private String message; + public static AlertResult success() { + return new AlertResult(true, null); + } + + public static AlertResult fail(String message) { + return new AlertResult(false, message); + } + } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java index 74c440fe76..f5cc938246 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java @@ -31,7 +31,7 @@ public final class DingTalkAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "ding talk params is null"); + return new AlertResult(false, "ding talk params is null"); } return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index c8ded8cfad..527e38cf77 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -126,7 +126,7 @@ public final class DingTalkSender { private AlertResult checkSendDingTalkSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (null == result) { alertResult.setMessage("send ding talk msg error"); @@ -140,7 +140,7 @@ public final class DingTalkSender { return alertResult; } if (sendMsgResponse.errcode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send ding talk msg success"); return alertResult; } @@ -164,7 +164,7 @@ public final class DingTalkSender { } catch (Exception e) { log.info("send ding talk alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send ding talk alert fail."); } return alertResult; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index cd30105c7a..90f64d7bb2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -52,7 +52,7 @@ public class DingTalkSenderTest { dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkSender = new DingTalkSender(dingTalkConfig); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertEquals(false, alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java index 5728461ae6..06aecd35db 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java @@ -35,24 +35,20 @@ public final class EmailAlertChannel implements AlertChannel { AlertData alert = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "mail params is null"); + return new AlertResult(false, "mail params is null"); } MailSender mailSender = new MailSender(paramsMap); AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent()); - boolean flag; - if (alertResult == null) { alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("alert send error."); log.info("alert send error : {}", alertResult.getMessage()); return alertResult; } - flag = Boolean.parseBoolean(String.valueOf(alertResult.getStatus())); - - if (flag) { + if (alertResult.isSuccess()) { log.info("alert send success"); alertResult.setMessage("email send success."); } else { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index 8826a44fba..58e1eb10b3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -154,7 +154,7 @@ public final class MailSender { */ public AlertResult sendMails(List receivers, List receiverCcs, String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); // if there is no receivers && no receiversCc, no need to process if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiverCcs)) { @@ -201,7 +201,7 @@ public final class MailSender { attachment(title, content, partContent); - alertResult.setStatus("true"); + alertResult.setSuccess(true); return alertResult; } catch (Exception e) { handleException(alertResult, e); @@ -380,7 +380,7 @@ public final class MailSender { email.setDebug(true); email.send(); - alertResult.setStatus("true"); + alertResult.setSuccess(true); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java index 9df19154aa..643cd8a01e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java @@ -66,7 +66,7 @@ public class EmailAlertChannelTest { alertInfo.setAlertParams(paramsMap); AlertResult alertResult = emailAlertChannel.process(alertInfo); Assertions.assertNotNull(alertResult); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } public String getEmailAlertParams() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index acc255ae0e..9a4b5e8257 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -77,7 +77,7 @@ public class MailUtilsTest { AlertResult alertResult = mailSender.sendMails( "Mysql Exception", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -107,7 +107,7 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd"); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails(title, content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } public String list2String() { @@ -142,24 +142,24 @@ public class MailUtilsTest { emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails(title, content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test - public void testAttachmentFile() throws Exception { + public void testAttachmentFile() { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails("gaojing", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test - public void testTableAttachmentFile() throws Exception { + public void testTableAttachmentFile() { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails("gaojing", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java index 8959c8aaec..29c78a9d1b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java @@ -31,7 +31,7 @@ public final class FeiShuAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "fei shu params is null"); + return new AlertResult(false, "fei shu params is null"); } return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index 369060843c..1c2f3656ea 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -80,7 +80,7 @@ public final class FeiShuSender { public static AlertResult checkSendFeiShuSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (org.apache.commons.lang3.StringUtils.isBlank(result)) { alertResult.setMessage("send fei shu msg error"); @@ -95,7 +95,7 @@ public final class FeiShuSender { return alertResult; } if (sendMsgResponse.statusCode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send fei shu msg success"); return alertResult; } @@ -136,7 +136,7 @@ public final class FeiShuSender { } catch (Exception e) { log.info("send fei shu alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send fei shu alert fail."); } return alertResult; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java index 41f372b85c..829b02dea6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java @@ -43,7 +43,7 @@ public class FeiShuSenderTest { alertData.setContent("feishu test content"); FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig); AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -87,12 +87,12 @@ public class FeiShuSenderTest { FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig); AlertResult alertResult = feiShuSender.checkSendFeiShuSendMsgResult(""); - Assertions.assertFalse(Boolean.valueOf(alertResult.getStatus())); + Assertions.assertFalse(alertResult.isSuccess()); AlertResult alertResult2 = feiShuSender.checkSendFeiShuSendMsgResult("123"); Assertions.assertEquals("send fei shu msg fail", alertResult2.getMessage()); String response = "{\"StatusCode\":\"0\",\"extra\":\"extra\",\"StatusMessage\":\"StatusMessage\"}"; AlertResult alertResult3 = feiShuSender.checkSendFeiShuSendMsgResult(response); - Assertions.assertTrue(Boolean.valueOf(alertResult3.getStatus())); + Assertions.assertTrue(alertResult3.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java index 944762f13f..caf1c4c598 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java @@ -31,7 +31,7 @@ public final class HttpAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "http params is null"); + return new AlertResult(false, "http params is null"); } return new HttpSender(paramsMap).send(alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java index a1de852407..e2a6606a39 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java @@ -92,18 +92,18 @@ public final class HttpSender { } if (httpRequest == null) { - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("Request types are not supported"); return alertResult; } try { String resp = this.getResponseString(httpRequest); - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage(resp); } catch (Exception e) { log.error("send http alert msg exception : {}", e.getMessage()); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage( String.format("Send http request alert failed: %s", e.getMessage())); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java index aebf6f9d50..ee67db47f1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java @@ -62,9 +62,9 @@ public class HttpAlertChannelTest { // HttpSender(paramsMap).send(alertData.getContent()); already test in HttpSenderTest.sendTest. so we can mock // it - doReturn(new AlertResult("true", "success")).when(alertChannel).process(any()); + doReturn(new AlertResult(true, "success")).when(alertChannel).process(any()); AlertResult alertResult = alertChannel.process(alertInfo); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); } /** diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java index be013457ac..40f589a10b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java @@ -46,7 +46,7 @@ public class HttpSenderTest { HttpSender httpSender = spy(new HttpSender(paramsMap)); doReturn("success").when(httpSender).getResponseString(any()); AlertResult alertResult = httpSender.send("Fault tolerance warning"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); Assertions.assertTrue(httpSender.getRequestUrl().contains(url)); Assertions.assertTrue(httpSender.getRequestUrl().contains(contentField)); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java index b033139520..430bacbf63 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java @@ -30,8 +30,8 @@ public final class PagerDutyAlertChannel implements AlertChannel { public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); - if (alertParams == null || alertParams.size() == 0) { - return new AlertResult("false", "PagerDuty alert params is empty"); + if (alertParams == null || alertParams.isEmpty()) { + return new AlertResult(false, "PagerDuty alert params is empty"); } return new PagerDutySender(alertParams).sendPagerDutyAlter(alertData.getTitle(), alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java index 65792c8eae..11dd01048a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java @@ -53,7 +53,7 @@ public final class PagerDutySender { public AlertResult sendPagerDutyAlter(String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send pager duty alert fail."); try { @@ -83,7 +83,7 @@ public final class PagerDutySender { String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8); try { if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send pager duty alert success"); } else { alertResult.setMessage( diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java index 16cf16f62f..52a47aa20e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java @@ -39,6 +39,6 @@ public class PagerDutySenderTest { public void testSend() { PagerDutySender pagerDutySender = new PagerDutySender(pagerDutyConfig); AlertResult alertResult = pagerDutySender.sendPagerDutyAlter("pagerduty test title", "pagerduty test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java index 7ca79253fa..5928faacec 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java @@ -31,7 +31,7 @@ public final class PrometheusAlertChannel implements AlertChannel { AlertData alertData = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "prometheus alert manager params is null"); + return new AlertResult(false, "prometheus alert manager params is null"); } return new PrometheusAlertSender(paramsMap).sendMessage(alertData); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java index 1106e6799f..48fda566b1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.alert.api.HttpServiceRetryStrategy; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -64,11 +65,10 @@ public class PrometheusAlertSender { String resp = sendMsg(alertData); return checkSendAlertManageMsgResult(resp); } catch (Exception e) { - String errorMsg = String.format("send prometheus alert manager alert error, exception: %s", e.getMessage()); - log.error(errorMsg); + log.error("Send prometheus alert manager alert error", e); alertResult = new AlertResult(); - alertResult.setStatus("false"); - alertResult.setMessage(errorMsg); + alertResult.setSuccess(false); + alertResult.setMessage(ExceptionUtils.getMessage(e)); } return alertResult; } @@ -106,10 +106,10 @@ public class PrometheusAlertSender { public AlertResult checkSendAlertManageMsgResult(String resp) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (Objects.equals(resp, PrometheusAlertConstants.ALERT_SUCCESS)) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("prometheus alert manager send success"); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java index 2347d97262..c0d18396e4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java @@ -55,17 +55,17 @@ public class PrometheusAlertSenderTest { " }]"); PrometheusAlertSender sender = new PrometheusAlertSender(config); AlertResult result = sender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @Test public void testCheckSendAlertManageMsgResult() { PrometheusAlertSender prometheusAlertSender = new PrometheusAlertSender(config); AlertResult alertResult1 = prometheusAlertSender.checkSendAlertManageMsgResult(""); - Assertions.assertFalse(Boolean.parseBoolean(alertResult1.getStatus())); + Assertions.assertFalse(alertResult1.isSuccess()); Assertions.assertEquals("prometheus alert manager send fail, resp is ", alertResult1.getMessage()); AlertResult alertResult2 = prometheusAlertSender.checkSendAlertManageMsgResult("alert success"); - Assertions.assertTrue(Boolean.parseBoolean(alertResult2.getStatus())); + Assertions.assertTrue(alertResult2.isSuccess()); Assertions.assertEquals("prometheus alert manager send success", alertResult2.getMessage()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java index d091eb9d82..81cd59a5a9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java @@ -33,7 +33,7 @@ public final class ScriptAlertChannel implements AlertChannel { AlertData alertData = alertinfo.getAlertData(); Map paramsMap = alertinfo.getAlertParams(); if (MapUtils.isEmpty(paramsMap)) { - return new AlertResult("false", "script params is empty"); + return new AlertResult(false, "script params is empty"); } return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle(), alertData.getContent()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java index a18adb2c7e..19b7149e74 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java @@ -56,7 +56,7 @@ public final class ScriptSender { } // If it is another type of alarm script can be added here, such as python - alertResult.setStatus("false"); + alertResult.setSuccess(false); log.error("script type error: {}", scriptType); alertResult.setMessage("script type error : " + scriptType); return alertResult; @@ -64,7 +64,7 @@ public final class ScriptSender { private AlertResult executeShellScript(String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (Boolean.TRUE.equals(OSUtils.isWindows())) { alertResult.setMessage("shell script not support windows os"); return alertResult; @@ -111,7 +111,7 @@ public final class ScriptSender { int exitCode = ProcessUtils.executeScript(cmd); if (exitCode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send script alert msg success"); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java index 32e996f5e8..c392b6f758 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.plugin.alert.script; +import static org.junit.jupiter.api.Assertions.assertFalse; + import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.HashMap; @@ -48,9 +50,9 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test title Kris", "test content"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); alertResult = scriptSender.sendScriptAlert("error msg title", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -58,7 +60,7 @@ public class ScriptSenderTest { scriptConfig.put(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, "' ; calc.exe ; '"); ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult = scriptSender.sendScriptAlert("test title Kris", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -67,7 +69,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test user params NPE", "test content"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); } @Test @@ -76,7 +78,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test path NPE", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -85,7 +87,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test path NPE", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + assertFalse(alertResult.isSuccess()); Assertions.assertTrue(alertResult.getMessage().contains("shell script is invalid, only support .sh file")); } @@ -95,7 +97,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test type is error", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java index c8cb36a78b..8052c7c4f1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java @@ -30,11 +30,11 @@ public final class SlackAlertChannel implements AlertChannel { public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); - if (alertParams == null || alertParams.size() == 0) { - return new AlertResult("false", "Slack alert params is empty"); + if (alertParams == null || alertParams.isEmpty()) { + return new AlertResult(false, "Slack alert params is empty"); } SlackSender slackSender = new SlackSender(alertParams); String response = slackSender.sendMessage(alertData.getTitle(), alertData.getContent()); - return new AlertResult("ok".equals(response) ? "true" : "false", response); + return new AlertResult("ok".equals(response), response); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java index efc8912d1a..ed33ef5497 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java @@ -30,7 +30,7 @@ public final class TelegramAlertChannel implements AlertChannel { public AlertResult process(AlertInfo info) { Map alertParams = info.getAlertParams(); if (alertParams == null || alertParams.isEmpty()) { - return new AlertResult("false", "Telegram alert params is empty"); + return AlertResult.fail("Telegram alert params is empty"); } AlertData data = info.getAlertData(); return new TelegramSender(alertParams).sendMessage(data); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 129bc62c1c..417e97d4cd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -105,7 +105,7 @@ public final class TelegramSender { } catch (Exception e) { log.warn("send telegram alert msg exception : {}", e.getMessage()); result = new AlertResult(); - result.setStatus("false"); + result.setSuccess(false); result.setMessage(String.format("send telegram alert fail. %s", e.getMessage())); } return result; @@ -113,7 +113,7 @@ public final class TelegramSender { private AlertResult parseRespToResult(String resp) { AlertResult result = new AlertResult(); - result.setStatus("false"); + result.setSuccess(false); if (null == resp || resp.isEmpty()) { result.setMessage("send telegram msg error. telegram server resp is empty"); return result; @@ -127,7 +127,7 @@ public final class TelegramSender { result.setMessage(String.format("send telegram alert fail. telegram server error_code: %d, description: %s", response.errorCode, response.description)); } else { - result.setStatus("true"); + result.setSuccess(true); result.setMessage("send telegram msg success."); } return result; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java index a57de30219..d05a45d73f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java @@ -52,7 +52,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, "XXXXXXX"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -65,7 +65,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, "-XXXXXXX"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @Test @@ -75,7 +75,7 @@ public class TelegramSenderTest { alertData.setContent("telegram test content"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -89,7 +89,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_MARKDOWN); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -102,7 +102,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_HTML); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java index 38a582f1c6..94f77aed6e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java @@ -33,7 +33,7 @@ public final class WebexTeamsAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); if (MapUtils.isEmpty(alertParams)) { - return new AlertResult("false", "WebexTeams alert params is empty"); + return new AlertResult(false, "WebexTeams alert params is empty"); } return new WebexTeamsSender(alertParams).sendWebexTeamsAlter(alertData); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java index f8201a40e0..3b8b3d21c8 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java @@ -67,7 +67,7 @@ public final class WebexTeamsSender { public AlertResult sendWebexTeamsAlter(AlertData alertData) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send webex teams alert fail."); try { @@ -93,7 +93,7 @@ public final class WebexTeamsSender { String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8); try { if (statusCode == HttpStatus.SC_OK) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send webex teams alert success"); } else { alertResult.setMessage(String.format( diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java index 1d3070cb55..ddc806e593 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java @@ -85,6 +85,6 @@ public class WebexTeamsSenderTest { public void testSend() { WebexTeamsSender webexTeamsSender = new WebexTeamsSender(webexTeamsConfig); AlertResult alertResult = webexTeamsSender.sendWebexTeamsAlter(alertData); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java index 786cdb159f..dcc53c7f59 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java @@ -31,7 +31,7 @@ public final class WeChatAlertChannel implements AlertChannel { AlertData alertData = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "we chat params is null"); + return new AlertResult(false, "we chat params is null"); } return new WeChatSender(paramsMap).sendEnterpriseWeChat(alertData.getTitle(), alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index c5ffec1f46..d3fba217dc 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -54,7 +54,6 @@ import lombok.extern.slf4j.Slf4j; public final class WeChatSender { private static final String MUST_NOT_NULL = " must not null"; - private static final String ALERT_STATUS = "false"; private static final String AGENT_ID_REG_EXP = "{agentId}"; private static final String MSG_REG_EXP = "{msg}"; private static final String USER_REG_EXP = "{toUser}"; @@ -178,7 +177,7 @@ public final class WeChatSender { private static AlertResult checkWeChatSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); if (null == result) { alertResult.setMessage("we chat send fail"); @@ -192,11 +191,11 @@ public final class WeChatSender { return alertResult; } if (sendMsgResponse.errcode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("we chat alert send success"); return alertResult; } - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); alertResult.setMessage(sendMsgResponse.getErrmsg()); return alertResult; } @@ -212,7 +211,7 @@ public final class WeChatSender { if (null == weChatToken) { alertResult = new AlertResult(); alertResult.setMessage("send we chat alert fail,get weChat token error"); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); return alertResult; } String enterpriseWeChatPushUrlReplace = ""; @@ -239,7 +238,7 @@ public final class WeChatSender { log.info("send we chat alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); alertResult.setMessage("send we chat alert fail"); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); } return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java index e0c934f436..6e4c318d13 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java @@ -71,7 +71,7 @@ public class WeChatSenderTest { WeChatSender weChatSender = new WeChatSender(weChatConfig); AlertResult alertResult = weChatSender.sendEnterpriseWeChat("test", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -79,7 +79,7 @@ public class WeChatSenderTest { weChatConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp()); WeChatSender weChatSender = new WeChatSender(weChatConfig); AlertResult alertResult = weChatSender.sendEnterpriseWeChat("test", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 507d7acb45..844a5983df 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -66,6 +66,12 @@ org.springframework.cloud spring-cloud-starter-kubernetes-client-config + + + org.springframework.boot + spring-boot-starter-test + test + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 55c5c3446c..2435711047 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -18,11 +18,7 @@ package org.apache.dolphinscheduler.alert; import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; -import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; -import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; -import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -50,14 +46,6 @@ public class AlertServer { @Autowired private AlertBootstrapService alertBootstrapService; - @Autowired - private ListenerEventPostService listenerEventPostService; - @Autowired - private AlertRpcServer alertRpcServer; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertRegistryClient alertRegistryClient; public static void main(String[] args) { AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); @@ -68,27 +56,14 @@ public class AlertServer { @PostConstruct public void run() { - log.info("Alert server is staring ..."); - alertPluginManager.start(); - alertRegistryClient.start(); + log.info("AlertServer is staring ..."); alertBootstrapService.start(); - listenerEventPostService.start(); - alertRpcServer.start(); - log.info("Alert server is started ..."); + log.info("AlertServer is started ..."); } @PreDestroy public void close() { - destroy("alert server destroy"); - } - - /** - * gracefully stop - * - * @param cause stop cause - */ - public void destroy(String cause) { - + String cause = "AlertServer destroy"; try { // set stop signal is true // execute only once @@ -96,19 +71,14 @@ public class AlertServer { log.warn("AlterServer is already stopped"); return; } - log.info("Alert server is stopping, cause: {}", cause); - try ( - AlertRpcServer closedAlertRpcServer = alertRpcServer; - AlertBootstrapService closedAlertBootstrapService = alertBootstrapService; - ListenerEventPostService closedListenerEventPostService = listenerEventPostService; - AlertRegistryClient closedAlertRegistryClient = alertRegistryClient) { - // close resource - } + log.info("AlertServer is stopping, cause: {}", cause); + alertBootstrapService.close(); // thread sleep 3 seconds for thread quietly stop ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis()); - log.info("Alter server stopped, cause: {}", cause); + log.info("AlertServer stopped, cause: {}", cause); } catch (Exception e) { - log.error("Alert server stop failed, cause: {}", cause, e); + log.error("AlertServer stop failed, cause: {}", cause, e); } } + } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 824851fd92..240f92b846 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -43,6 +43,8 @@ public final class AlertConfig implements Validator { private Duration maxHeartbeatInterval = Duration.ofSeconds(60); + private int senderParallelism = 100; + private String alertServerAddress; @Override @@ -58,6 +60,10 @@ public final class AlertConfig implements Validator { errors.rejectValue("max-heartbeat-interval", null, "should be a valid duration"); } + if (senderParallelism <= 0) { + errors.rejectValue("sender-parallelism", null, "should be a positive number"); + } + if (StringUtils.isEmpty(alertServerAddress)) { alertConfig.setAlertServerAddress(NetUtils.getAddr(alertConfig.getPort())); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java index db75a49371..606834a2e9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java @@ -45,6 +45,12 @@ public class AlertServerMetrics { .register(Metrics.globalRegistry); } + public void registerSendingAlertGauge(final Supplier supplier) { + Gauge.builder("ds.alert.sending", supplier) + .description("Number of sending alert") + .register(Metrics.globalRegistry); + } + public static void registerUncachedException(final Supplier supplier) { Gauge.builder("ds.alert.uncached.exception", supplier) .description("number of uncached exception") diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java index 1035018e9c..badd463166 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java @@ -36,8 +36,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -@Component @Slf4j +@Component public final class AlertPluginManager { private final PluginDao pluginDao; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java index 0bfefed223..a5481fdd49 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.alert.registry; import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat; import org.apache.dolphinscheduler.common.model.BaseHeartBeatTask; @@ -42,12 +43,15 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask private final RegistryClient registryClient; private final MetricsProvider metricsProvider; + + private final AlertHAServer alertHAServer; private final String heartBeatPath; private final long startupTime; public AlertHeartbeatTask(AlertConfig alertConfig, MetricsProvider metricsProvider, - RegistryClient registryClient) { + RegistryClient registryClient, + AlertHAServer alertHAServer) { super("AlertHeartbeatTask", alertConfig.getMaxHeartbeatInterval().toMillis()); this.startupTime = System.currentTimeMillis(); this.alertConfig = alertConfig; @@ -55,6 +59,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask this.registryClient = registryClient; this.heartBeatPath = RegistryNodeType.ALERT_SERVER.getRegistryPath() + "/" + alertConfig.getAlertServerAddress(); + this.alertHAServer = alertHAServer; this.processId = OSUtils.getProcessID(); } @@ -70,6 +75,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) .serverStatus(ServerStatus.NORMAL) + .isActive(alertHAServer.isActive()) .host(NetUtils.getHost()) .port(alertConfig.getPort()) .build(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java index 616220bd1b..1b7839d816 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.alert.registry; import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.registry.api.RegistryClient; -import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import lombok.extern.slf4j.Slf4j; @@ -42,10 +42,12 @@ public class AlertRegistryClient implements AutoCloseable { private AlertHeartbeatTask alertHeartbeatTask; + @Autowired + private AlertHAServer alertHAServer; + public void start() { log.info("AlertRegistryClient starting..."); - registryClient.getLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); - alertHeartbeatTask = new AlertHeartbeatTask(alertConfig, metricsProvider, registryClient); + alertHeartbeatTask = new AlertHeartbeatTask(alertConfig, metricsProvider, registryClient, alertHAServer); alertHeartbeatTask.start(); // start heartbeat task log.info("AlertRegistryClient started..."); @@ -55,7 +57,6 @@ public class AlertRegistryClient implements AutoCloseable { public void close() { log.info("AlertRegistryClient closing..."); alertHeartbeatTask.shutdown(); - registryClient.releaseLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); log.info("AlertRegistryClient closed..."); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java index 9f11fa6c2e..6a5ed3e0be 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.alert.rpc; -import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; +import org.apache.dolphinscheduler.alert.service.AlertSender; import org.apache.dolphinscheduler.extract.alert.IAlertOperator; import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest; import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; @@ -32,16 +32,15 @@ import org.springframework.stereotype.Service; public class AlertOperatorImpl implements IAlertOperator { @Autowired - private AlertBootstrapService alertBootstrapService; + private AlertSender alertSender; @Override public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) { log.info("Received AlertSendRequest : {}", alertSendRequest); - AlertSendResponse alertSendResponse = alertBootstrapService.syncHandler( + AlertSendResponse alertSendResponse = alertSender.syncHandler( alertSendRequest.getGroupId(), alertSendRequest.getTitle(), - alertSendRequest.getContent(), - alertSendRequest.getWarnType()); + alertSendRequest.getContent()); log.info("Handle AlertSendRequest finish: {}", alertSendResponse); return alertSendResponse; } @@ -49,7 +48,7 @@ public class AlertOperatorImpl implements IAlertOperator { @Override public AlertSendResponse sendTestAlert(AlertTestSendRequest alertSendRequest) { log.info("Received AlertTestSendRequest : {}", alertSendRequest); - AlertSendResponse alertSendResponse = alertBootstrapService.syncTestSend( + AlertSendResponse alertSendResponse = alertSender.syncTestSend( alertSendRequest.getPluginDefineId(), alertSendRequest.getPluginInstanceParams()); log.info("Handle AlertTestSendRequest finish: {}", alertSendResponse); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java new file mode 100644 index 0000000000..1c61659ce3 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractEventFetcher extends BaseDaemonThread implements EventFetcher { + + protected static final int FETCH_SIZE = 100; + + protected static final long FETCH_INTERVAL = 5_000; + + protected final AlertHAServer alertHAServer; + + private final EventPendingQueue eventPendingQueue; + + private final AtomicBoolean runningFlag = new AtomicBoolean(false); + + private Integer eventOffset; + + protected AbstractEventFetcher(String fetcherName, + AlertHAServer alertHAServer, + EventPendingQueue eventPendingQueue) { + super(fetcherName); + this.alertHAServer = alertHAServer; + this.eventPendingQueue = eventPendingQueue; + this.eventOffset = -1; + } + + @Override + public synchronized void start() { + if (!runningFlag.compareAndSet(false, true)) { + throw new IllegalArgumentException("AlertEventFetcher is already started"); + } + log.info("AlertEventFetcher starting..."); + super.start(); + log.info("AlertEventFetcher started..."); + } + + @Override + public void run() { + while (runningFlag.get()) { + try { + if (!alertHAServer.isActive()) { + log.debug("The current node is not active, will not loop Alert"); + Thread.sleep(FETCH_INTERVAL); + continue; + } + List pendingEvents = fetchPendingEvent(eventOffset); + if (CollectionUtils.isEmpty(pendingEvents)) { + log.debug("No pending events found"); + Thread.sleep(FETCH_INTERVAL); + continue; + } + for (T alert : pendingEvents) { + eventPendingQueue.put(alert); + } + eventOffset = Math.max(eventOffset, + pendingEvents.stream().map(this::getEventOffset).max(Integer::compareTo).get()); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + } catch (Exception ex) { + log.error("AlertEventFetcher error", ex); + } + } + } + + protected abstract int getEventOffset(T event); + + @Override + public void shutdown() { + if (!runningFlag.compareAndSet(true, false)) { + log.warn("The AlertEventFetcher is not started"); + } + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java new file mode 100644 index 0000000000..568125002e --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractEventLoop extends BaseDaemonThread implements EventLoop { + + private final EventPendingQueue eventPendingQueue; + + private final AtomicInteger handlingEventCount; + + private final int eventHandleWorkerNum; + + private final ThreadPoolExecutor threadPoolExecutor; + + private final AtomicBoolean runningFlag = new AtomicBoolean(false); + + protected AbstractEventLoop(String name, + ThreadPoolExecutor threadPoolExecutor, + EventPendingQueue eventPendingQueue) { + super(name); + this.handlingEventCount = new AtomicInteger(0); + this.eventHandleWorkerNum = threadPoolExecutor.getMaximumPoolSize(); + this.threadPoolExecutor = threadPoolExecutor; + this.eventPendingQueue = eventPendingQueue; + } + + @Override + public synchronized void start() { + if (!runningFlag.compareAndSet(false, true)) { + throw new IllegalArgumentException(getClass().getName() + " is already started"); + } + log.info("{} starting...", getClass().getName()); + super.start(); + log.info("{} started...", getClass().getName()); + } + + @Override + public void run() { + while (runningFlag.get()) { + try { + if (handlingEventCount.get() >= eventHandleWorkerNum) { + log.debug("There is no idle event worker, waiting for a while..."); + Thread.sleep(1000); + continue; + } + T pendingEvent = eventPendingQueue.take(); + handlingEventCount.incrementAndGet(); + CompletableFuture.runAsync(() -> handleEvent(pendingEvent), threadPoolExecutor) + .whenComplete((aVoid, throwable) -> { + if (throwable != null) { + log.error("Handle event: {} error", pendingEvent, throwable); + } + handlingEventCount.decrementAndGet(); + }); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + log.error("Loop event thread has been interrupted..."); + break; + } catch (Exception ex) { + log.error("Loop event error", ex); + } + } + } + + @Override + public int getHandlingEventCount() { + return handlingEventCount.get(); + } + + @Override + public void shutdown() { + if (!runningFlag.compareAndSet(true, false)) { + log.warn(getClass().getName() + " is not started"); + } + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java new file mode 100644 index 0000000000..1d7e213ab9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import java.util.concurrent.LinkedBlockingQueue; + +public abstract class AbstractEventPendingQueue implements EventPendingQueue { + + private final LinkedBlockingQueue pendingAlertQueue; + + private final int capacity; + + protected AbstractEventPendingQueue(int capacity) { + this.capacity = capacity; + this.pendingAlertQueue = new LinkedBlockingQueue<>(capacity); + } + + @Override + public void put(T alert) throws InterruptedException { + pendingAlertQueue.put(alert); + } + + @Override + public T take() throws InterruptedException { + return pendingAlertQueue.take(); + } + + @Override + public int size() { + return pendingAlertQueue.size(); + } + + @Override + public int capacity() { + return capacity; + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java new file mode 100644 index 0000000000..deff97da49 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.alert.api.AlertChannel; +import org.apache.dolphinscheduler.alert.api.AlertConstants; +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertInfo; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; +import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +@Slf4j +public abstract class AbstractEventSender implements EventSender { + + protected final AlertPluginManager alertPluginManager; + + private final long sendEventTimeout; + + protected AbstractEventSender(AlertPluginManager alertPluginManager, long sendEventTimeout) { + this.alertPluginManager = alertPluginManager; + this.sendEventTimeout = sendEventTimeout; + } + + @Override + public void sendEvent(T event) { + List alertPluginInstanceList = getAlertPluginInstanceList(event); + if (CollectionUtils.isEmpty(alertPluginInstanceList)) { + onError(event, "No bind plugin instance found"); + return; + } + AlertData alertData = getAlertData(event); + List alertSendStatuses = new ArrayList<>(); + for (AlertPluginInstance instance : alertPluginInstanceList) { + AlertResult alertResult = doSendEvent(instance, alertData); + AlertStatus alertStatus = + alertResult.isSuccess() ? AlertStatus.EXECUTION_SUCCESS : AlertStatus.EXECUTION_FAILURE; + AlertSendStatus alertSendStatus = AlertSendStatus.builder() + .alertId(getEventId(event)) + .alertPluginInstanceId(instance.getId()) + .sendStatus(alertStatus) + .log(JSONUtils.toJsonString(alertResult)) + .createTime(new Date()) + .build(); + alertSendStatuses.add(alertSendStatus); + } + long failureCount = alertSendStatuses.stream() + .map(alertSendStatus -> alertSendStatus.getSendStatus() == AlertStatus.EXECUTION_FAILURE) + .count(); + long successCount = alertSendStatuses.stream() + .map(alertSendStatus -> alertSendStatus.getSendStatus() == AlertStatus.EXECUTION_SUCCESS) + .count(); + if (successCount == 0) { + onError(event, JSONUtils.toJsonString(alertSendStatuses)); + } else { + if (failureCount > 0) { + onPartialSuccess(event, JSONUtils.toJsonString(alertSendStatuses)); + } else { + onSuccess(event, JSONUtils.toJsonString(alertSendStatuses)); + } + } + } + + public abstract List getAlertPluginInstanceList(T event); + + public abstract AlertData getAlertData(T event); + + public abstract Integer getEventId(T event); + + public abstract void onError(T event, String log); + + public abstract void onPartialSuccess(T event, String log); + + public abstract void onSuccess(T event, String log); + + @Override + public AlertResult doSendEvent(AlertPluginInstance instance, AlertData alertData) { + int pluginDefineId = instance.getPluginDefineId(); + Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); + if (!alertChannelOptional.isPresent()) { + return AlertResult.fail("Cannot find the alertPlugin: " + pluginDefineId); + } + AlertChannel alertChannel = alertChannelOptional.get(); + + AlertInfo alertInfo = AlertInfo.builder() + .alertData(alertData) + .alertParams(PluginParamsTransfer.getPluginParamsMap(instance.getPluginInstanceParams())) + .alertPluginInstanceId(instance.getId()) + .build(); + try { + AlertResult alertResult; + if (sendEventTimeout <= 0) { + if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { + alertResult = alertChannel.closeAlert(alertInfo); + } else { + alertResult = alertChannel.process(alertInfo); + } + } else { + CompletableFuture future; + if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { + future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); + } else { + future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); + } + alertResult = future.get(sendEventTimeout, TimeUnit.MILLISECONDS); + } + checkNotNull(alertResult, "AlertResult cannot be null"); + return alertResult; + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + return AlertResult.fail(ExceptionUtils.getMessage(interruptedException)); + } catch (Exception e) { + log.error("Send alert data {} failed", alertData, e); + return AlertResult.fail(ExceptionUtils.getMessage(e)); + } + } + + @Override + public AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams) { + + Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); + if (!alertChannelOptional.isPresent()) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + AlertSendResponse.AlertSendResponseResult.fail("Cannot find the alertPlugin: " + pluginDefineId); + return AlertSendResponse.fail(Lists.newArrayList(alertSendResponseResult)); + } + AlertData alertData = AlertData.builder() + .title(AlertConstants.TEST_TITLE) + .content(AlertConstants.TEST_CONTENT) + .build(); + + AlertInfo alertInfo = AlertInfo.builder() + .alertData(alertData) + .alertParams(PluginParamsTransfer.getPluginParamsMap(pluginInstanceParams)) + .build(); + + try { + AlertResult alertResult = alertChannelOptional.get().process(alertInfo); + Preconditions.checkNotNull(alertResult, "AlertResult cannot be null"); + if (alertResult.isSuccess()) { + return AlertSendResponse + .success(Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.success())); + } + return AlertSendResponse.fail( + Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.fail(alertResult.getMessage()))); + } catch (Exception e) { + log.error("Test send alert error", e); + return new AlertSendResponse(false, + Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.fail(ExceptionUtils.getMessage(e)))); + } + + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java index 77e62a65a0..5553e01bc9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java @@ -17,350 +17,84 @@ package org.apache.dolphinscheduler.alert.service; -import org.apache.dolphinscheduler.alert.api.AlertChannel; -import org.apache.dolphinscheduler.alert.api.AlertConstants; -import org.apache.dolphinscheduler.alert.api.AlertData; -import org.apache.dolphinscheduler.alert.api.AlertInfo; -import org.apache.dolphinscheduler.alert.api.AlertResult; -import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.entity.Alert; -import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; -import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; -import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nullable; +import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; +import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import com.google.common.collect.Lists; - -@Service +/** + * The bootstrap service for alert server. it will start all the necessary component for alert server. + */ @Slf4j -public final class AlertBootstrapService extends BaseDaemonThread implements AutoCloseable { +@Service +public final class AlertBootstrapService implements AutoCloseable { - @Autowired - private AlertDao alertDao; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertConfig alertConfig; + private final AlertRpcServer alertRpcServer; - public AlertBootstrapService() { - super("AlertBootstrapService"); + private final AlertRegistryClient alertRegistryClient; + + private final AlertPluginManager alertPluginManager; + + private final AlertHAServer alertHAServer; + + private final AlertEventFetcher alertEventFetcher; + + private final AlertEventLoop alertEventLoop; + + private final ListenerEventLoop listenerEventLoop; + + private final ListenerEventFetcher listenerEventFetcher; + + public AlertBootstrapService(AlertRpcServer alertRpcServer, + AlertRegistryClient alertRegistryClient, + AlertPluginManager alertPluginManager, + AlertHAServer alertHAServer, + AlertEventFetcher alertEventFetcher, + AlertEventLoop alertEventLoop, + ListenerEventLoop listenerEventLoop, + ListenerEventFetcher listenerEventFetcher) { + this.alertRpcServer = alertRpcServer; + this.alertRegistryClient = alertRegistryClient; + this.alertPluginManager = alertPluginManager; + this.alertHAServer = alertHAServer; + this.alertEventFetcher = alertEventFetcher; + this.alertEventLoop = alertEventLoop; + this.listenerEventLoop = listenerEventLoop; + this.listenerEventFetcher = listenerEventFetcher; } - @Override - public void run() { - log.info("Alert sender thread started"); - while (!ServerLifeCycleManager.isStopped()) { - try { - List alerts = alertDao.listPendingAlerts(); - if (CollectionUtils.isEmpty(alerts)) { - log.debug("There is not waiting alerts"); - continue; - } - AlertServerMetrics.registerPendingAlertGauge(alerts::size); - this.send(alerts); - } catch (Exception e) { - log.error("Alert sender thread meet an exception", e); - } finally { - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 5L); - } - } - log.info("Alert sender thread stopped"); - } + public void start() { + log.info("AlertBootstrapService starting..."); + alertPluginManager.start(); + alertRpcServer.start(); + alertRegistryClient.start(); + alertHAServer.start(); - public void send(List alerts) { - for (Alert alert : alerts) { - // get alert group from alert - int alertId = alert.getId(); - int alertGroupId = Optional.ofNullable(alert.getAlertGroupId()).orElse(0); - List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); - if (CollectionUtils.isEmpty(alertInstanceList)) { - log.error("send alert msg fail,no bind plugin instance."); - List alertResults = Lists.newArrayList(new AlertResult("false", - "no bind plugin instance")); - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, JSONUtils.toJsonString(alertResults), alertId); - continue; - } - AlertData alertData = AlertData.builder() - .id(alertId) - .content(alert.getContent()) - .log(alert.getLog()) - .title(alert.getTitle()) - .warnType(alert.getWarningType().getCode()) - .alertType(alert.getAlertType().getCode()) - .build(); + listenerEventFetcher.start(); + alertEventFetcher.start(); - int sendSuccessCount = 0; - List alertSendStatuses = new ArrayList<>(); - List alertResults = new ArrayList<>(); - for (AlertPluginInstance instance : alertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertStatus sendStatus = Boolean.parseBoolean(alertResult.getStatus()) - ? AlertStatus.EXECUTION_SUCCESS - : AlertStatus.EXECUTION_FAILURE; - AlertSendStatus alertSendStatus = AlertSendStatus.builder() - .alertId(alertId) - .alertPluginInstanceId(instance.getId()) - .sendStatus(sendStatus) - .log(JSONUtils.toJsonString(alertResult)) - .createTime(new Date()) - .build(); - alertSendStatuses.add(alertSendStatus); - if (AlertStatus.EXECUTION_SUCCESS.equals(sendStatus)) { - sendSuccessCount++; - AlertServerMetrics.incAlertSuccessCount(); - } else { - AlertServerMetrics.incAlertFailCount(); - } - alertResults.add(alertResult); - } - } - AlertStatus alertStatus = AlertStatus.EXECUTION_SUCCESS; - if (sendSuccessCount == 0) { - alertStatus = AlertStatus.EXECUTION_FAILURE; - } else if (sendSuccessCount < alertInstanceList.size()) { - alertStatus = AlertStatus.EXECUTION_PARTIAL_SUCCESS; - } - // we update the alert first to avoid duplicate key in alertSendStatus - // this may loss the alertSendStatus if the server restart - // todo: use transaction to update these two table - alertDao.updateAlert(alertStatus, JSONUtils.toJsonString(alertResults), alertId); - alertDao.insertAlertSendStatus(alertSendStatuses); - } - } - - /** - * sync send alert handler - * - * @param alertGroupId alertGroupId - * @param title title - * @param content content - * @return AlertSendResponseCommand - */ - public AlertSendResponse syncHandler(int alertGroupId, String title, String content, int warnType) { - List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); - AlertData alertData = AlertData.builder() - .content(content) - .title(title) - .warnType(warnType) - .build(); - - boolean sendResponseStatus = true; - List sendResponseResults = new ArrayList<>(); - - if (CollectionUtils.isEmpty(alertInstanceList)) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - String message = String.format("Alert GroupId %s send error : not found alert instance", alertGroupId); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(message); - sendResponseResults.add(alertSendResponseResult); - log.error("Alert GroupId {} send error : not found alert instance", alertGroupId); - return new AlertSendResponse(false, sendResponseResults); - } - - for (AlertPluginInstance instance : alertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult( - Boolean.parseBoolean(alertResult.getStatus()), - alertResult.getMessage()); - sendResponseStatus = sendResponseStatus && alertSendResponseResult.isSuccess(); - sendResponseResults.add(alertSendResponseResult); - } - } - - return new AlertSendResponse(sendResponseStatus, sendResponseResults); - } - - /** - * alert result handler - * - * @param instance instance - * @param alertData alertData - * @return AlertResult - */ - private @Nullable AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) { - String pluginInstanceName = instance.getInstanceName(); - int pluginDefineId = instance.getPluginDefineId(); - Optional alertChannelOptional = alertPluginManager.getAlertChannel(instance.getPluginDefineId()); - if (!alertChannelOptional.isPresent()) { - String message = String.format("Alert Plugin %s send error: the channel doesn't exist, pluginDefineId: %s", - pluginInstanceName, - pluginDefineId); - log.error("Alert Plugin {} send error : not found plugin {}", pluginInstanceName, pluginDefineId); - return new AlertResult("false", message); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams()); - String instanceWarnType = WarningType.ALL.getDescp(); - - if (MapUtils.isNotEmpty(paramsMap)) { - instanceWarnType = paramsMap.getOrDefault(AlertConstants.NAME_WARNING_TYPE, WarningType.ALL.getDescp()); - } - - WarningType warningType = WarningType.of(instanceWarnType); - - if (warningType == null) { - String message = String.format("Alert Plugin %s send error : plugin warnType is null", pluginInstanceName); - log.error("Alert Plugin {} send error : plugin warnType is null", pluginInstanceName); - return new AlertResult("false", message); - } - - boolean sendWarning = false; - switch (warningType) { - case ALL: - sendWarning = true; - break; - case SUCCESS: - if (alertData.getWarnType() == WarningType.SUCCESS.getCode()) { - sendWarning = true; - } - break; - case FAILURE: - if (alertData.getWarnType() == WarningType.FAILURE.getCode()) { - sendWarning = true; - } - break; - default: - } - - if (!sendWarning) { - String message = String.format( - "Alert Plugin %s send ignore warning type not match: plugin warning type is %s, alert data warning type is %s", - pluginInstanceName, warningType.getCode(), alertData.getWarnType()); - log.info( - "Alert Plugin {} send ignore warning type not match: plugin warning type is {}, alert data warning type is {}", - pluginInstanceName, warningType.getCode(), alertData.getWarnType()); - return new AlertResult("false", message); - } - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .alertPluginInstanceId(instance.getId()) - .build(); - int waitTimeout = alertConfig.getWaitTimeout(); - try { - AlertResult alertResult; - if (waitTimeout <= 0) { - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - alertResult = alertChannel.closeAlert(alertInfo); - } else { - alertResult = alertChannel.process(alertInfo); - } - } else { - CompletableFuture future; - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); - } else { - future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); - } - alertResult = future.get(waitTimeout, TimeUnit.MILLISECONDS); - } - if (alertResult == null) { - throw new RuntimeException("Alert result cannot be null"); - } - return alertResult; - } catch (InterruptedException e) { - log.error("send alert error alert data id :{},", alertData.getId(), e); - Thread.currentThread().interrupt(); - return new AlertResult("false", e.getMessage()); - } catch (Exception e) { - log.error("send alert error alert data id :{},", alertData.getId(), e); - return new AlertResult("false", e.getMessage()); - } - } - - public AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams) { - - boolean sendResponseStatus = true; - List sendResponseResults = new ArrayList<>(); - - Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); - if (!alertChannelOptional.isPresent()) { - String message = String.format("Test send alert error: the channel doesn't exist, pluginDefineId: %s", - pluginDefineId); - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(message); - sendResponseResults.add(alertSendResponseResult); - log.error("Test send alert error : not found plugin {}", pluginDefineId); - return new AlertSendResponse(false, sendResponseResults); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = PluginParamsTransfer.getPluginParamsMap(pluginInstanceParams); - - AlertData alertData = AlertData.builder() - .title(AlertConstants.TEST_TITLE) - .content(AlertConstants.TEST_CONTENT) - .warnType(WarningType.ALL.getCode()) - .build(); - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .build(); - - try { - AlertResult alertResult = alertChannel.process(alertInfo); - if (alertResult != null) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult( - Boolean.parseBoolean(alertResult.getStatus()), - alertResult.getMessage()); - sendResponseStatus = alertSendResponseResult.isSuccess(); - sendResponseResults.add(alertSendResponseResult); - } - } catch (Exception e) { - log.error("Test send alert error", e); - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(e.getMessage()); - sendResponseResults.add(alertSendResponseResult); - return new AlertSendResponse(false, sendResponseResults); - } - - return new AlertSendResponse(sendResponseStatus, sendResponseResults); + listenerEventLoop.start(); + alertEventLoop.start(); + log.info("AlertBootstrapService started..."); } @Override public void close() { - log.info("Closed AlertBootstrapService..."); - } + log.info("AlertBootstrapService stopping..."); + try ( + AlertRpcServer closedAlertRpcServer = alertRpcServer; + AlertRegistryClient closedAlertRegistryClient = alertRegistryClient) { + // close resource + listenerEventFetcher.shutdown(); + alertEventFetcher.shutdown(); + listenerEventLoop.shutdown(); + alertEventLoop.shutdown(); + alertHAServer.shutdown(); + } + log.info("AlertBootstrapService stopped..."); + } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java new file mode 100644 index 0000000000..11a668ae1d --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertEventFetcher extends AbstractEventFetcher { + + private final AlertDao alertDao; + + public AlertEventFetcher(AlertHAServer alertHAServer, + AlertDao alertDao, + AlertEventPendingQueue alertEventPendingQueue) { + super("AlertEventFetcher", alertHAServer, alertEventPendingQueue); + this.alertDao = alertDao; + } + + @Override + public List fetchPendingEvent(int eventOffset) { + return alertDao.listPendingAlerts(eventOffset); + } + + @Override + protected int getEventOffset(Alert event) { + return event.getId(); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java new file mode 100644 index 0000000000..e975f1ad51 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertEventLoop extends AbstractEventLoop { + + private final AlertSender alertSender; + + public AlertEventLoop(AlertEventPendingQueue alertEventPendingQueue, + AlertSenderThreadPoolFactory alertSenderThreadPoolFactory, + AlertSender alertSender) { + super("AlertEventLoop", alertSenderThreadPoolFactory.getThreadPool(), alertEventPendingQueue); + this.alertSender = alertSender; + AlertServerMetrics.registerPendingAlertGauge(this::getHandlingEventCount); + } + + @Override + public void handleEvent(Alert event) { + alertSender.sendEvent(event); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java new file mode 100644 index 0000000000..17fe7ccd0b --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import org.springframework.stereotype.Component; + +@Component +public class AlertEventPendingQueue extends AbstractEventPendingQueue { + + public AlertEventPendingQueue(AlertConfig alertConfig) { + super(alertConfig.getSenderParallelism() * 3 + 1); + AlertServerMetrics.registerPendingAlertGauge(this::size); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java new file mode 100644 index 0000000000..998bc655c4 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; +import org.apache.dolphinscheduler.registry.api.ha.AbstractHAServer; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertHAServer extends AbstractHAServer { + + public AlertHAServer(Registry registry) { + super(registry, RegistryNodeType.ALERT_LOCK.getRegistryPath()); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java new file mode 100644 index 0000000000..9c9cd034bd --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertSender extends AbstractEventSender { + + private final AlertDao alertDao; + + public AlertSender(AlertDao alertDao, + AlertPluginManager alertPluginManager, + AlertConfig alertConfig) { + super(alertPluginManager, alertConfig.getWaitTimeout()); + this.alertDao = alertDao; + } + + /** + * sync send alert handler + * + * @param alertGroupId alertGroupId + * @param title title + * @param content content + * @return AlertSendResponseCommand + */ + public AlertSendResponse syncHandler(int alertGroupId, String title, String content) { + List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); + AlertData alertData = AlertData.builder() + .content(content) + .title(title) + .build(); + + boolean sendResponseStatus = true; + List sendResponseResults = new ArrayList<>(); + + if (CollectionUtils.isEmpty(alertInstanceList)) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + new AlertSendResponse.AlertSendResponseResult(); + String message = String.format("Alert GroupId %s send error : not found alert instance", alertGroupId); + alertSendResponseResult.setSuccess(false); + alertSendResponseResult.setMessage(message); + sendResponseResults.add(alertSendResponseResult); + log.error("Alert GroupId {} send error : not found alert instance", alertGroupId); + return new AlertSendResponse(false, sendResponseResults); + } + + for (AlertPluginInstance instance : alertInstanceList) { + AlertResult alertResult = doSendEvent(instance, alertData); + if (alertResult != null) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + new AlertSendResponse.AlertSendResponseResult( + alertResult.isSuccess(), + alertResult.getMessage()); + sendResponseStatus = sendResponseStatus && alertSendResponseResult.isSuccess(); + sendResponseResults.add(alertSendResponseResult); + } + } + + return new AlertSendResponse(sendResponseStatus, sendResponseResults); + } + + @Override + public List getAlertPluginInstanceList(Alert event) { + return alertDao.listInstanceByAlertGroupId(event.getAlertGroupId()); + } + + @Override + public AlertData getAlertData(Alert event) { + return AlertData.builder() + .id(event.getId()) + .content(event.getContent()) + .log(event.getLog()) + .title(event.getTitle()) + .alertType(event.getAlertType().getCode()) + .build(); + } + + @Override + public Integer getEventId(Alert event) { + return event.getId(); + } + + @Override + public void onError(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, log, event.getId()); + } + + @Override + public void onPartialSuccess(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_PARTIAL_SUCCESS, log, event.getId()); + } + + @Override + public void onSuccess(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, log, event.getId()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java new file mode 100644 index 0000000000..fd8c731b17 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.springframework.stereotype.Component; + +@Component +public class AlertSenderThreadPoolFactory { + + private final ThreadPoolExecutor threadPool; + + public AlertSenderThreadPoolFactory(AlertConfig alertConfig) { + this.threadPool = ThreadUtils.newDaemonFixedThreadExecutor("AlertSenderThread", + alertConfig.getSenderParallelism()); + } + + public ThreadPoolExecutor getThreadPool() { + return threadPool; + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java new file mode 100644 index 0000000000..089fb4edc9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import java.util.List; + +/** + * The interface responsible for fetching events. + * + * @param the type of event + */ +public interface EventFetcher { + + void start(); + + List fetchPendingEvent(int eventOffset); + + void shutdown(); +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java new file mode 100644 index 0000000000..04219f99ae --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +/** + * The interface responsible for consuming event from upstream, e.g {@link EventPendingQueue}. + * + * @param the type of event + */ +public interface EventLoop { + + /** + * Start the event loop, once the event loop is started, it will keep consuming event from upstream. + */ + void start(); + + /** + * Handle the given event. + */ + void handleEvent(T event); + + /** + * Get the count of handling event. + */ + int getHandlingEventCount(); + + /** + * Shutdown the event loop, once the event loop is shutdown, it will stop consuming event from upstream. + */ + void shutdown(); + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java new file mode 100644 index 0000000000..c8538138bc --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +/** + * The interface responsible for managing pending events. + * + * @param the type of event + */ +public interface EventPendingQueue { + + void put(T alert) throws InterruptedException; + + T take() throws InterruptedException; + + int size(); + + int capacity(); +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java new file mode 100644 index 0000000000..04bc85e573 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; + +public interface EventSender { + + void sendEvent(T event); + + AlertResult doSendEvent(AlertPluginInstance instance, AlertData alertData); + + AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams); + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java new file mode 100644 index 0000000000..57549d7e97 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class ListenerEventFetcher extends AbstractEventFetcher { + + private final ListenerEventDao listenerEventDao; + + protected ListenerEventFetcher(AlertHAServer alertHAServer, + ListenerEventDao listenerEventDao, + ListenerEventPendingQueue listenerEventPendingQueue) { + super("ListenerEventFetcher", alertHAServer, listenerEventPendingQueue); + this.listenerEventDao = listenerEventDao; + } + + @Override + protected int getEventOffset(ListenerEvent event) { + return event.getId(); + } + + @Override + public List fetchPendingEvent(int eventOffset) { + return listenerEventDao.listingPendingEvents(eventOffset, FETCH_SIZE); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java new file mode 100644 index 0000000000..f1c00967f9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import org.springframework.stereotype.Component; + +@Component +public class ListenerEventLoop extends AbstractEventLoop { + + private final ListenerEventSender listenerEventSender; + + protected ListenerEventLoop(AlertSenderThreadPoolFactory alertSenderThreadPoolFactory, + ListenerEventSender listenerEventSender, + ListenerEventPendingQueue listenerEventPendingQueue) { + super("ListenerEventLoop", alertSenderThreadPoolFactory.getThreadPool(), listenerEventPendingQueue); + this.listenerEventSender = listenerEventSender; + } + + @Override + public void handleEvent(ListenerEvent event) { + listenerEventSender.sendEvent(event); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java new file mode 100644 index 0000000000..47d0c77dff --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import org.springframework.stereotype.Component; + +@Component +public class ListenerEventPendingQueue extends AbstractEventPendingQueue { + + public ListenerEventPendingQueue(AlertConfig alertConfig) { + super(alertConfig.getSenderParallelism() * 3 + 1); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java deleted file mode 100644 index b57562c711..0000000000 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.alert.service; - -import org.apache.dolphinscheduler.alert.api.AlertChannel; -import org.apache.dolphinscheduler.alert.api.AlertData; -import org.apache.dolphinscheduler.alert.api.AlertInfo; -import org.apache.dolphinscheduler.alert.api.AlertResult; -import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; -import org.apache.dolphinscheduler.dao.entity.ListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.AbstractListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionCreatedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionDeletedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionUpdatedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessEndListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessFailListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessStartListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskEndListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskFailListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskStartListenerEvent; -import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.curator.shaded.com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nullable; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -@Service -@Slf4j -public final class ListenerEventPostService extends BaseDaemonThread implements AutoCloseable { - - @Value("${alert.query_alert_threshold:100}") - private Integer QUERY_ALERT_THRESHOLD; - @Autowired - private ListenerEventMapper listenerEventMapper; - @Autowired - private AlertPluginInstanceMapper alertPluginInstanceMapper; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertConfig alertConfig; - - public ListenerEventPostService() { - super("ListenerEventPostService"); - } - - @Override - public void run() { - log.info("listener event post thread started"); - while (!ServerLifeCycleManager.isStopped()) { - try { - List listenerEvents = listenerEventMapper - .listingListenerEventByStatus(AlertStatus.WAIT_EXECUTION, QUERY_ALERT_THRESHOLD); - if (CollectionUtils.isEmpty(listenerEvents)) { - log.debug("There is no waiting listener events"); - continue; - } - this.send(listenerEvents); - } catch (Exception e) { - log.error("listener event post thread meet an exception", e); - } finally { - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 5L); - } - } - log.info("listener event post thread stopped"); - } - - public void send(List listenerEvents) { - for (ListenerEvent listenerEvent : listenerEvents) { - int eventId = listenerEvent.getId(); - List globalAlertInstanceList = - alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList(); - if (CollectionUtils.isEmpty(globalAlertInstanceList)) { - log.error("post listener event fail,no bind global plugin instance."); - listenerEventMapper.updateListenerEvent(eventId, AlertStatus.EXECUTION_FAILURE, - "no bind plugin instance", new Date()); - continue; - } - AbstractListenerEvent event = generateEventFromContent(listenerEvent); - if (event == null) { - log.error("parse listener event to abstract listener event fail.ed {}", listenerEvent.getContent()); - listenerEventMapper.updateListenerEvent(eventId, AlertStatus.EXECUTION_FAILURE, - "parse listener event to abstract listener event failed", new Date()); - continue; - } - List events = Lists.newArrayList(event); - AlertData alertData = AlertData.builder() - .id(eventId) - .content(JSONUtils.toJsonString(events)) - .log(listenerEvent.getLog()) - .title(event.getTitle()) - .warnType(WarningType.GLOBAL.getCode()) - .alertType(event.getEventType().getCode()) - .build(); - - int sendSuccessCount = 0; - List failedPostResults = new ArrayList<>(); - for (AlertPluginInstance instance : globalAlertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertStatus sendStatus = Boolean.parseBoolean(alertResult.getStatus()) - ? AlertStatus.EXECUTION_SUCCESS - : AlertStatus.EXECUTION_FAILURE; - if (AlertStatus.EXECUTION_SUCCESS.equals(sendStatus)) { - sendSuccessCount++; - } else { - AlertSendStatus alertSendStatus = AlertSendStatus.builder() - .alertId(eventId) - .alertPluginInstanceId(instance.getId()) - .sendStatus(sendStatus) - .log(JSONUtils.toJsonString(alertResult)) - .createTime(new Date()) - .build(); - failedPostResults.add(alertSendStatus); - } - } - } - if (sendSuccessCount == globalAlertInstanceList.size()) { - listenerEventMapper.deleteById(eventId); - } else { - AlertStatus alertStatus = - sendSuccessCount == 0 ? AlertStatus.EXECUTION_FAILURE : AlertStatus.EXECUTION_PARTIAL_SUCCESS; - listenerEventMapper.updateListenerEvent(eventId, alertStatus, JSONUtils.toJsonString(failedPostResults), - new Date()); - } - } - } - - /** - * alert result handler - * - * @param instance instance - * @param alertData alertData - * @return AlertResult - */ - private @Nullable AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) { - String pluginInstanceName = instance.getInstanceName(); - int pluginDefineId = instance.getPluginDefineId(); - Optional alertChannelOptional = alertPluginManager.getAlertChannel(instance.getPluginDefineId()); - if (!alertChannelOptional.isPresent()) { - String message = - String.format("Global Alert Plugin %s send error: the channel doesn't exist, pluginDefineId: %s", - pluginInstanceName, - pluginDefineId); - log.error("Global Alert Plugin {} send error : not found plugin {}", pluginInstanceName, pluginDefineId); - return new AlertResult("false", message); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams()); - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .alertPluginInstanceId(instance.getId()) - .build(); - int waitTimeout = alertConfig.getWaitTimeout(); - try { - AlertResult alertResult; - if (waitTimeout <= 0) { - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - alertResult = alertChannel.closeAlert(alertInfo); - } else { - alertResult = alertChannel.process(alertInfo); - } - } else { - CompletableFuture future; - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); - } else { - future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); - } - alertResult = future.get(waitTimeout, TimeUnit.MILLISECONDS); - } - if (alertResult == null) { - throw new RuntimeException("Alert result cannot be null"); - } - return alertResult; - } catch (InterruptedException e) { - log.error("post listener event error alert data id :{},", alertData.getId(), e); - Thread.currentThread().interrupt(); - return new AlertResult("false", e.getMessage()); - } catch (Exception e) { - log.error("post listener event error alert data id :{},", alertData.getId(), e); - return new AlertResult("false", e.getMessage()); - } - } - - private AbstractListenerEvent generateEventFromContent(ListenerEvent listenerEvent) { - String content = listenerEvent.getContent(); - switch (listenerEvent.getEventType()) { - case SERVER_DOWN: - return JSONUtils.parseObject(content, ServerDownListenerEvent.class); - case PROCESS_DEFINITION_CREATED: - return JSONUtils.parseObject(content, ProcessDefinitionCreatedListenerEvent.class); - case PROCESS_DEFINITION_UPDATED: - return JSONUtils.parseObject(content, ProcessDefinitionUpdatedListenerEvent.class); - case PROCESS_DEFINITION_DELETED: - return JSONUtils.parseObject(content, ProcessDefinitionDeletedListenerEvent.class); - case PROCESS_START: - return JSONUtils.parseObject(content, ProcessStartListenerEvent.class); - case PROCESS_END: - return JSONUtils.parseObject(content, ProcessEndListenerEvent.class); - case PROCESS_FAIL: - return JSONUtils.parseObject(content, ProcessFailListenerEvent.class); - case TASK_START: - return JSONUtils.parseObject(content, TaskStartListenerEvent.class); - case TASK_END: - return JSONUtils.parseObject(content, TaskEndListenerEvent.class); - case TASK_FAIL: - return JSONUtils.parseObject(content, TaskFailListenerEvent.class); - default: - return null; - } - } - @Override - public void close() { - log.info("Closed ListenerEventPostService..."); - } -} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java new file mode 100644 index 0000000000..7d06500edd --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.AbstractListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionCreatedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionDeletedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionUpdatedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessEndListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessFailListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessStartListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskEndListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskFailListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskStartListenerEvent; +import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import org.apache.curator.shaded.com.google.common.collect.Lists; + +import java.util.Date; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class ListenerEventSender extends AbstractEventSender { + + private final ListenerEventDao listenerEventDao; + + private final AlertPluginInstanceMapper alertPluginInstanceMapper; + + public ListenerEventSender(ListenerEventDao listenerEventDao, + AlertPluginInstanceMapper alertPluginInstanceMapper, + AlertPluginManager alertPluginManager, + AlertConfig alertConfig) { + super(alertPluginManager, alertConfig.getWaitTimeout()); + this.listenerEventDao = listenerEventDao; + this.alertPluginInstanceMapper = alertPluginInstanceMapper; + } + + private AbstractListenerEvent generateEventFromContent(ListenerEvent listenerEvent) { + String content = listenerEvent.getContent(); + AbstractListenerEvent event = null; + switch (listenerEvent.getEventType()) { + case SERVER_DOWN: + event = JSONUtils.parseObject(content, ServerDownListenerEvent.class); + break; + case PROCESS_DEFINITION_CREATED: + event = JSONUtils.parseObject(content, ProcessDefinitionCreatedListenerEvent.class); + break; + case PROCESS_DEFINITION_UPDATED: + event = JSONUtils.parseObject(content, ProcessDefinitionUpdatedListenerEvent.class); + break; + case PROCESS_DEFINITION_DELETED: + event = JSONUtils.parseObject(content, ProcessDefinitionDeletedListenerEvent.class); + break; + case PROCESS_START: + event = JSONUtils.parseObject(content, ProcessStartListenerEvent.class); + break; + case PROCESS_END: + event = JSONUtils.parseObject(content, ProcessEndListenerEvent.class); + break; + case PROCESS_FAIL: + event = JSONUtils.parseObject(content, ProcessFailListenerEvent.class); + break; + case TASK_START: + event = JSONUtils.parseObject(content, TaskStartListenerEvent.class); + break; + case TASK_END: + event = JSONUtils.parseObject(content, TaskEndListenerEvent.class); + break; + case TASK_FAIL: + event = JSONUtils.parseObject(content, TaskFailListenerEvent.class); + break; + default: + throw new IllegalArgumentException("Unsupported event type: " + listenerEvent.getEventType()); + } + if (event == null) { + throw new IllegalArgumentException("Failed to parse event from content: " + content); + } + return event; + } + + @Override + public List getAlertPluginInstanceList(ListenerEvent event) { + return alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList(); + } + + @Override + public AlertData getAlertData(ListenerEvent listenerEvent) { + AbstractListenerEvent event = generateEventFromContent(listenerEvent); + return AlertData.builder() + .id(listenerEvent.getId()) + .content(JSONUtils.toJsonString(Lists.newArrayList(event))) + .log(listenerEvent.getLog()) + .title(event.getTitle()) + .alertType(event.getEventType().getCode()) + .build(); + } + + @Override + public Integer getEventId(ListenerEvent event) { + return event.getId(); + } + + @Override + public void onError(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_FAILURE, log, new Date()); + } + + @Override + public void onPartialSuccess(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_PARTIAL_SUCCESS, log, new Date()); + } + + @Override + public void onSuccess(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_FAILURE, log, new Date()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 0dbb6988ce..6fbcc04feb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -73,7 +73,8 @@ alert: # Define value is (0 = infinite), and alert server would be waiting alert result. wait-timeout: 0 max-heartbeat-interval: 60s - query_alert_threshold: 100 + # The maximum number of alerts that can be processed in parallel + sender-parallelism: 100 registry: type: zookeeper diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java new file mode 100644 index 0000000000..1a72f0a5c9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.config; + +import static com.google.common.truth.Truth.assertThat; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; + +@AutoConfigureMockMvc +@SpringBootTest(classes = AlertConfig.class) +class AlertConfigTest { + + @Autowired + private AlertConfig alertConfig; + + @Test + void testValidate() { + assertThat(alertConfig.getWaitTimeout()).isEqualTo(10); + assertThat(alertConfig.getMaxHeartbeatInterval()).isEqualTo(Duration.ofSeconds(59)); + assertThat(alertConfig.getSenderParallelism()).isEqualTo(101); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java similarity index 73% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index eafba16585..400afd34dc 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -24,15 +24,13 @@ import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; +import org.apache.dolphinscheduler.alert.service.AlertSender; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.ListenerEvent; -import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; @@ -42,19 +40,20 @@ import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AlertBootstrapServiceTest { +@ExtendWith(MockitoExtension.class) +class AlertSenderTest { - private static final Logger logger = LoggerFactory.getLogger(AlertBootstrapServiceTest.class); + private static final Logger logger = LoggerFactory.getLogger(AlertSenderTest.class); @Mock private AlertDao alertDao; @@ -66,7 +65,7 @@ public class AlertBootstrapServiceTest { private AlertConfig alertConfig; @InjectMocks - private AlertBootstrapService alertBootstrapService; + private AlertSender alertSender; private static final String PLUGIN_INSTANCE_PARAMS = "{\"User\":\"xx\",\"receivers\":\"xx\",\"sender\":\"xx\",\"smtpSslTrust\":\"*\",\"enableSmtpAuth\":\"true\",\"receiverCcs\":null,\"showType\":\"table\",\"starttlsEnable\":\"false\",\"serverPort\":\"25\",\"serverHost\":\"xx\",\"Password\":\"xx\",\"sslEnable\":\"false\"}"; @@ -74,25 +73,17 @@ public class AlertBootstrapServiceTest { private static final String PLUGIN_INSTANCE_NAME = "alert-instance-mail"; private static final String TITLE = "alert mail test TITLE"; private static final String CONTENT = "alert mail test CONTENT"; - private static final List EVENTS = new ArrayList<>(); private static final int PLUGIN_DEFINE_ID = 1; private static final int ALERT_GROUP_ID = 1; - @BeforeEach - public void before() { - MockitoAnnotations.initMocks(this); - } - @Test - public void testSyncHandler() { + void testSyncHandler() { // 1.alert instance does not exist when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(null); - when(alertConfig.getWaitTimeout()).thenReturn(0); - AlertSendResponse alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + AlertSendResponse alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -108,12 +99,7 @@ public class AlertBootstrapServiceTest { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList); - String pluginName = "alert-plugin-mail"; - PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null); - when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); - - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -122,37 +108,32 @@ public class AlertBootstrapServiceTest { AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(null); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - when(alertConfig.getWaitTimeout()).thenReturn(0); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); // 4.abnormal information inside the alert plug-in code AlertResult alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(false)); + alertResult.setSuccess(false); alertResult.setMessage("Abnormal information inside the alert plug-in code"); when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); // 5.alert plugin send success alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(true)); + alertResult.setSuccess(true); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - when(alertConfig.getWaitTimeout()).thenReturn(5000); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertTrue(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -160,17 +141,13 @@ public class AlertBootstrapServiceTest { } @Test - public void testRun() { - List alertList = new ArrayList<>(); + void testRun() { Alert alert = new Alert(); alert.setId(1); alert.setAlertGroupId(ALERT_GROUP_ID); alert.setTitle(TITLE); alert.setContent(CONTENT); alert.setWarningType(WarningType.FAILURE); - alertList.add(alert); - - // alertSenderService = new AlertSenderService(); int pluginDefineId = 1; String pluginInstanceParams = "alert-instance-mail-params"; @@ -181,25 +158,18 @@ public class AlertBootstrapServiceTest { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(alertInstanceList); - String pluginName = "alert-plugin-mail"; - PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null); - when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); - AlertResult alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(true)); + alertResult.setSuccess(true); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); - AlertChannel alertChannelMock = mock(AlertChannel.class); - when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); - when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertTrue(Boolean.parseBoolean(alertResult.getStatus())); + Assertions.assertTrue(alertResult.isSuccess()); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(new ArrayList<>()); - alertBootstrapService.send(alertList); + alertSender.sendEvent(alert); } @Test - public void testSendAlert() { + void testSendAlert() { AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(true)); + sendResult.setSuccess(true); sendResult.setMessage(String.format("Alert Plugin %s send success", PLUGIN_INSTANCE_NAME)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); @@ -209,6 +179,6 @@ public class AlertBootstrapServiceTest { Mockito.mockStatic(PluginParamsTransfer.class); pluginParamsTransferMockedStatic.when(() -> PluginParamsTransfer.getPluginParamsMap(PLUGIN_INSTANCE_PARAMS)) .thenReturn(paramsMap); - alertBootstrapService.syncTestSend(PLUGIN_DEFINE_ID, PLUGIN_INSTANCE_PARAMS); + alertSender.syncTestSend(PLUGIN_DEFINE_ID, PLUGIN_INSTANCE_PARAMS); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java similarity index 82% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java index 33917267f0..0304be022c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; +import org.apache.dolphinscheduler.alert.service.ListenerEventSender; import org.apache.dolphinscheduler.common.enums.AlertPluginInstanceType; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.ListenerEventType; @@ -33,7 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.ListenerEvent; import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; import org.apache.commons.codec.digest.DigestUtils; @@ -43,39 +43,32 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.mockito.junit.jupiter.MockitoExtension; -public class ListenerEventPostServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(ListenerEventPostServiceTest.class); +@ExtendWith(MockitoExtension.class) +class ListenerEventSenderTest { @Mock - private ListenerEventMapper listenerEventMapper; + private ListenerEventDao listenerEventDao; + @Mock private AlertPluginInstanceMapper alertPluginInstanceMapper; @Mock private AlertPluginManager alertPluginManager; + @Mock private AlertConfig alertConfig; @InjectMocks - private ListenerEventPostService listenerEventPostService; - - @BeforeEach - public void before() { - MockitoAnnotations.initMocks(this); - } + private ListenerEventSender listenerEventSender; @Test - public void testSendServerDownEventSuccess() { - List events = new ArrayList<>(); + void testSendServerDownEventSuccess() { ServerDownListenerEvent serverDownListenerEvent = new ServerDownListenerEvent(); serverDownListenerEvent.setEventTime(new Date()); serverDownListenerEvent.setType("WORKER"); @@ -88,7 +81,6 @@ public class ListenerEventPostServiceTest { successEvent.setEventType(ListenerEventType.SERVER_DOWN); successEvent.setCreateTime(new Date()); successEvent.setUpdateTime(new Date()); - events.add(successEvent); int pluginDefineId = 1; String pluginInstanceParams = @@ -103,19 +95,17 @@ public class ListenerEventPostServiceTest { when(alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList()).thenReturn(alertInstanceList); AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(true)); + sendResult.setSuccess(true); sendResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertTrue(Boolean.parseBoolean(sendResult.getStatus())); - when(listenerEventMapper.deleteById(1)).thenReturn(1); - listenerEventPostService.send(events); + Assertions.assertTrue(sendResult.isSuccess()); + listenerEventSender.sendEvent(successEvent); } @Test - public void testSendServerDownEventFailed() { - List events = new ArrayList<>(); + void testSendServerDownEventFailed() { ServerDownListenerEvent serverDownListenerEvent = new ServerDownListenerEvent(); serverDownListenerEvent.setEventTime(new Date()); serverDownListenerEvent.setType("WORKER"); @@ -128,7 +118,6 @@ public class ListenerEventPostServiceTest { successEvent.setEventType(ListenerEventType.SERVER_DOWN); successEvent.setCreateTime(new Date()); successEvent.setUpdateTime(new Date()); - events.add(successEvent); int pluginDefineId = 1; String pluginInstanceParams = @@ -143,12 +132,12 @@ public class ListenerEventPostServiceTest { when(alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList()).thenReturn(alertInstanceList); AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(false)); + sendResult.setSuccess(false); sendResult.setMessage(String.format("Alert Plugin %s send failed", pluginInstanceName)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertFalse(Boolean.parseBoolean(sendResult.getStatus())); - listenerEventPostService.send(events); + Assertions.assertFalse(sendResult.isSuccess()); + listenerEventSender.sendEvent(successEvent); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java new file mode 100644 index 0000000000..a643e50e76 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.truth.Truth.assertThat; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +import lombok.SneakyThrows; + +import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AlertEventPendingQueueTest { + + private AlertEventPendingQueue alertEventPendingQueue; + + private static final int QUEUE_SIZE = 10; + + @BeforeEach + public void before() { + AlertConfig alertConfig = new AlertConfig(); + alertConfig.setSenderParallelism(QUEUE_SIZE); + this.alertEventPendingQueue = new AlertEventPendingQueue(alertConfig); + } + + @SneakyThrows + @Test + void put() { + for (int i = 0; i < alertEventPendingQueue.capacity(); i++) { + alertEventPendingQueue.put(new Alert()); + } + + CompletableFuture completableFuture = CompletableFuture.runAsync(() -> { + try { + alertEventPendingQueue.put(new Alert()); + System.out.println(alertEventPendingQueue.size()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + assertThrowsExactly(ConditionTimeoutException.class, + () -> await() + .timeout(Duration.ofSeconds(2)) + .until(completableFuture::isDone)); + + } + + @Test + void take() { + CompletableFuture completableFuture = CompletableFuture.runAsync(() -> { + try { + alertEventPendingQueue.take(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + assertThrowsExactly(ConditionTimeoutException.class, + () -> await() + .timeout(Duration.ofSeconds(2)) + .until(completableFuture::isDone)); + } + + @SneakyThrows + @Test + void size() { + for (int i = 0; i < alertEventPendingQueue.capacity(); i++) { + alertEventPendingQueue.put(new Alert()); + assertThat(alertEventPendingQueue.size()).isEqualTo(i + 1); + } + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java new file mode 100644 index 0000000000..50f44fb43f --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AlertSenderThreadPoolFactoryTest { + + private final AlertConfig alertConfig = new AlertConfig(); + + private final AlertSenderThreadPoolFactory alertSenderThreadPoolFactory = + new AlertSenderThreadPoolFactory(alertConfig); + + @Test + void getThreadPool() { + ThreadPoolExecutor threadPool = alertSenderThreadPoolFactory.getThreadPool(); + assertThat(threadPool.getCorePoolSize()).isEqualTo(alertConfig.getSenderParallelism()); + assertThat(threadPool.getMaximumPoolSize()).isEqualTo(alertConfig.getSenderParallelism()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml new file mode 100644 index 0000000000..d16d05a678 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + profiles: + active: postgresql + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + banner: + charset: UTF-8 + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + +server: + port: 50053 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +alert: + port: 50052 + # Mark each alert of alert server if late after x milliseconds as failed. + # Define value is (0 = infinite), and alert server would be waiting alert result. + wait-timeout: 10 + max-heartbeat-interval: 59s + # The maximum number of alerts that can be processed in parallel + sender-parallelism: 101 + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + +metrics: + enabled: true + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java index 1e4f49721a..afa7e97023 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java @@ -20,6 +20,6 @@ package org.apache.dolphinscheduler.common.enums; public enum ServerStatus { NORMAL, - BUSY + BUSY, } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java index 9faaef82be..8533ca6e48 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java @@ -24,4 +24,9 @@ import lombok.experimental.SuperBuilder; @NoArgsConstructor public class AlertServerHeartBeat extends BaseHeartBeat implements HeartBeat { + /** + * If the alert server is active or standby + */ + private boolean isActive; + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 3b71312d0f..d9f6109834 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -63,8 +63,7 @@ import com.google.common.collect.Lists; @Slf4j public class AlertDao { - @Value("${alert.query_alert_threshold:100}") - private Integer QUERY_ALERT_THRESHOLD; + private static final Integer QUERY_ALERT_THRESHOLD = 100; @Value("${alert.alarm-suppression.crash:60}") private Integer crashAlarmSuppression; @@ -104,8 +103,8 @@ public class AlertDao { * update alert sending(execution) status * * @param alertStatus alertStatus - * @param log alert results json - * @param id id + * @param log alert results json + * @param id id * @return update alert result */ public int updateAlert(AlertStatus alertStatus, String log, int id) { @@ -134,9 +133,9 @@ public class AlertDao { /** * add AlertSendStatus * - * @param sendStatus alert send status - * @param log log - * @param alertId alert id + * @param sendStatus alert send status + * @param log log + * @param alertId alert id * @param alertPluginInstanceId alert plugin instance id * @return insert count */ @@ -192,7 +191,7 @@ public class AlertDao { * process time out alert * * @param processInstance processInstance - * @param projectUser projectUser + * @param projectUser projectUser */ public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProjectUser projectUser) { int alertGroupId = processInstance.getWarningGroupId(); @@ -238,8 +237,8 @@ public class AlertDao { * task timeout warn * * @param processInstance processInstanceId - * @param taskInstance taskInstance - * @param projectUser projectUser + * @param taskInstance taskInstance + * @param projectUser projectUser */ public void sendTaskTimeoutAlert(ProcessInstance processInstance, TaskInstance taskInstance, ProjectUser projectUser) { @@ -271,10 +270,11 @@ public class AlertDao { } /** - * List alerts that are pending for execution + * List pending alerts which id > minAlertId and status = {@link AlertStatus#WAIT_EXECUTION} order by id asc. */ - public List listPendingAlerts() { - return alertMapper.listingAlertByStatus(AlertStatus.WAIT_EXECUTION.getCode(), QUERY_ALERT_THRESHOLD); + public List listPendingAlerts(int minAlertId) { + return alertMapper.listingAlertByStatus(minAlertId, AlertStatus.WAIT_EXECUTION.getCode(), + QUERY_ALERT_THRESHOLD); } public List listAlerts(int processInstanceId) { @@ -283,15 +283,6 @@ public class AlertDao { return alertMapper.selectList(wrapper); } - /** - * for test - * - * @return AlertMapper - */ - public AlertMapper getAlertMapper() { - return alertMapper; - } - /** * list all alert plugin instance by alert group id * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java index c30c1c9043..aab7b6f5f2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java @@ -34,9 +34,10 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; public interface AlertMapper extends BaseMapper { /** - * Query the alert by alertStatus and return limit with default sort. + * Query the alert which id > minAlertId and status = alertStatus order by id asc. */ - List listingAlertByStatus(@Param("alertStatus") int alertStatus, @Param("limit") int limit); + List listingAlertByStatus(@Param("minAlertId") int minAlertId, @Param("alertStatus") int alertStatus, + @Param("limit") int limit); /** * Insert server crash alert diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java index 820ac3b3a6..f3e187f803 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java @@ -34,7 +34,8 @@ public interface ListenerEventMapper extends BaseMapper { void insertServerDownEvent(@Param("event") ListenerEvent event, @Param("crashAlarmSuppressionStartTime") Date crashAlarmSuppressionStartTime); - List listingListenerEventByStatus(@Param("postStatus") AlertStatus postStatus, + List listingListenerEventByStatus(@Param("minId") int minId, + @Param("postStatus") int postStatus, @Param("limit") int limit); void updateListenerEvent(@Param("eventId") int eventId, @Param("postStatus") AlertStatus postStatus, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java new file mode 100644 index 0000000000..424c616cd3 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import java.util.Date; +import java.util.List; + +public interface ListenerEventDao extends IDao { + + List listingPendingEvents(int minId, int limit); + + void updateListenerEvent(int eventId, AlertStatus alertStatus, String message, Date date); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java new file mode 100644 index 0000000000..06c4dccd5e --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.Date; +import java.util.List; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Repository; + +@Slf4j +@Repository +public class ListenerEventDaoImpl extends BaseDao implements ListenerEventDao { + + public ListenerEventDaoImpl(@NonNull ListenerEventMapper listenerEventMapper) { + super(listenerEventMapper); + } + + @Override + public List listingPendingEvents(int minId, int limit) { + return mybatisMapper.listingListenerEventByStatus(minId, AlertStatus.WAIT_EXECUTION.getCode(), limit); + } + + @Override + public void updateListenerEvent(int eventId, AlertStatus alertStatus, String message, Date date) { + mybatisMapper.updateListenerEvent(eventId, alertStatus, message, date); + } +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index e56afa830e..f0c32aae78 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -55,7 +55,9 @@ select from t_ds_alert - where alert_status = #{alertStatus} + where id > #{minAlertId} + and alert_status = #{alertStatus} + order by id asc limit #{limit} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml index ae76bfcadd..8e6d3bbc3a 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml @@ -60,7 +60,8 @@ select from t_ds_listener_event - where post_status = #{postStatus.code} + where id > #{minId} and post_status = #{postStatus} + order by id asc limit #{limit} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java index f3e877aaa3..4e239265c4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java @@ -81,7 +81,7 @@ public class ListenerEventMapperTest extends BaseDaoTest { ListenerEvent event2 = generateServerDownListenerEvent("192.168.x.2"); listenerEventMapper.batchInsert(Lists.newArrayList(event1, event2)); List listenerEvents = - listenerEventMapper.listingListenerEventByStatus(AlertStatus.WAIT_EXECUTION, 50); + listenerEventMapper.listingListenerEventByStatus(-1, AlertStatus.WAIT_EXECUTION.getCode(), 50); Assertions.assertEquals(listenerEvents.size(), 2); } @@ -111,8 +111,10 @@ public class ListenerEventMapperTest extends BaseDaoTest { ListenerEvent actualAlert = listenerEventMapper.selectById(event.getId()); Assertions.assertNull(actualAlert); } + /** * create server down event + * * @param host worker host * @return listener event */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index fe8545f3e2..a2c2c2ab98 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -18,37 +18,23 @@ package org.apache.dolphinscheduler.dao.repository.impl; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.ProfileType; import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.Alert; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.transaction.annotation.Transactional; -@ActiveProfiles(ProfileType.H2) -@ExtendWith(MockitoExtension.class) -@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) -@SpringBootTest(classes = DaoConfiguration.class) -@Transactional -@Rollback -public class AlertDaoTest { +class AlertDaoTest extends BaseDaoTest { @Autowired private AlertDao alertDao; @Test - public void testAlertDao() { + void testAlertDao() { Alert alert = new Alert(); alert.setTitle("Mysql Exception"); alert.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " @@ -57,25 +43,25 @@ public class AlertDaoTest { alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); alertDao.addAlert(alert); - List alerts = alertDao.listPendingAlerts(); + List alerts = alertDao.listPendingAlerts(-1); Assertions.assertNotNull(alerts); Assertions.assertNotEquals(0, alerts.size()); } @Test - public void testAddAlertSendStatus() { + void testAddAlertSendStatus() { int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS, "success", 1, 1); Assertions.assertEquals(1, insertCount); } @Test - public void testSendServerStoppedAlert() { + void testSendServerStoppedAlert() { int alertGroupId = 1; String host = "127.0.0.998165432"; String serverType = "Master"; alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); - long count = alertDao.listPendingAlerts() + long count = alertDao.listPendingAlerts(-1) .stream() .filter(alert -> alert.getContent().contains(host)) .count(); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java new file mode 100644 index 0000000000..2574c52f78 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.ListenerEventType; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.Date; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class ListenerEventDaoImplTest extends BaseDaoTest { + + @Autowired + private ListenerEventDao listenerEventDao; + + @Test + void listingPendingEvents() { + int minId = -1; + int limit = 10; + assertThat(listenerEventDao.listingPendingEvents(minId, limit)).isEmpty(); + + ListenerEvent listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.WAIT_EXECUTION) + .build(); + listenerEventDao.insert(listenerEvent); + + listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.EXECUTION_SUCCESS) + .build(); + listenerEventDao.insert(listenerEvent); + + assertThat(listenerEventDao.listingPendingEvents(minId, limit)).hasSize(1); + } + + @Test + void updateListenerEvent() { + ListenerEvent listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.WAIT_EXECUTION) + .build(); + listenerEventDao.insert(listenerEvent); + listenerEventDao.updateListenerEvent(listenerEvent.getId(), AlertStatus.EXECUTION_SUCCESS, "test", new Date()); + assertThat(listenerEventDao.queryById(listenerEvent.getId()).getPostStatus()) + .isEqualTo(AlertStatus.EXECUTION_SUCCESS); + } +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java index 832f3e1fab..e1db2a233e 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java @@ -37,6 +37,14 @@ public class AlertSendResponse { private List resResults; + public static AlertSendResponse success(List resResults) { + return new AlertSendResponse(true, resResults); + } + + public static AlertSendResponse fail(List resResults) { + return new AlertSendResponse(false, resResults); + } + @Data @NoArgsConstructor @AllArgsConstructor @@ -46,6 +54,14 @@ public class AlertSendResponse { private String message; + public static AlertSendResponseResult success() { + return new AlertSendResponseResult(true, null); + } + + public static AlertSendResponseResult fail(String message) { + return new AlertSendResponseResult(false, message); + } + } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java index 8bdb8b9021..86b82a8fb6 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java @@ -58,7 +58,6 @@ public interface Registry extends Closeable { String get(String key); /** - * * @param key * @param value * @param deleteOnDisconnect if true, when the connection state is disconnected, the key will be deleted @@ -67,6 +66,7 @@ public interface Registry extends Closeable { /** * This function will delete the keys whose prefix is {@param key} + * * @param key the prefix of deleted key * @throws if the key not exists, there is a registryException */ @@ -90,6 +90,11 @@ public interface Registry extends Closeable { */ boolean acquireLock(String key); + /** + * Acquire the lock of the prefix {@param key}, if acquire in the given timeout return true, else return false. + */ + boolean acquireLock(String key, long timeout); + /** * Release the lock of the prefix {@param key} */ diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java new file mode 100644 index 0000000000..5dca5552b3 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.Registry; + +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import com.google.common.collect.Lists; + +@Slf4j +public abstract class AbstractHAServer implements HAServer { + + private final Registry registry; + + private final String serverPath; + + private ServerStatus serverStatus; + + private final List serverStatusChangeListeners; + + public AbstractHAServer(Registry registry, String serverPath) { + this.registry = registry; + this.serverPath = serverPath; + this.serverStatus = ServerStatus.STAND_BY; + this.serverStatusChangeListeners = Lists.newArrayList(new DefaultServerStatusChangeListener()); + } + + @Override + public void start() { + registry.subscribe(serverPath, event -> { + if (Event.Type.REMOVE.equals(event.type())) { + if (isActive() && !participateElection()) { + statusChange(ServerStatus.STAND_BY); + } + } + }); + ScheduledExecutorService electionSelectionThread = + ThreadUtils.newSingleDaemonScheduledExecutorService("election-selection-thread"); + electionSelectionThread.schedule(() -> { + if (isActive()) { + return; + } + if (participateElection()) { + statusChange(ServerStatus.ACTIVE); + } + }, 10, TimeUnit.SECONDS); + } + + @Override + public boolean isActive() { + return ServerStatus.ACTIVE.equals(getServerStatus()); + } + + @Override + public boolean participateElection() { + return registry.acquireLock(serverPath, 3_000); + } + + @Override + public void addServerStatusChangeListener(ServerStatusChangeListener listener) { + serverStatusChangeListeners.add(listener); + } + + @Override + public ServerStatus getServerStatus() { + return serverStatus; + } + + @Override + public void shutdown() { + if (isActive()) { + registry.releaseLock(serverPath); + } + } + + private void statusChange(ServerStatus targetStatus) { + synchronized (this) { + ServerStatus originStatus = serverStatus; + serverStatus = targetStatus; + serverStatusChangeListeners.forEach(listener -> listener.change(originStatus, serverStatus)); + } + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java new file mode 100644 index 0000000000..f2e332ea20 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractServerStatusChangeListener implements ServerStatusChangeListener { + + @Override + public void change(HAServer.ServerStatus originStatus, HAServer.ServerStatus currentStatus) { + log.info("The status change from {} to {}.", originStatus, currentStatus); + if (originStatus == HAServer.ServerStatus.ACTIVE) { + if (currentStatus == HAServer.ServerStatus.STAND_BY) { + changeToStandBy(); + } + } else if (originStatus == HAServer.ServerStatus.STAND_BY) { + if (currentStatus == HAServer.ServerStatus.ACTIVE) { + changeToActive(); + } + } + } + + public abstract void changeToActive(); + + public abstract void changeToStandBy(); +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java new file mode 100644 index 0000000000..d2acbcb516 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DefaultServerStatusChangeListener extends AbstractServerStatusChangeListener { + + @Override + public void changeToActive() { + log.info("The status is active now."); + } + + @Override + public void changeToStandBy() { + log.info("The status is standby now."); + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java new file mode 100644 index 0000000000..6a79e6eb84 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +/** + * Interface for HA server, used to select a active server from multiple servers. + * In HA mode, there are multiple servers, only one server is active, others are standby. + */ +public interface HAServer { + + /** + * Start the server. + */ + void start(); + + /** + * Judge whether the server is active. + * + * @return true if the current server is active. + */ + boolean isActive(); + + /** + * Participate in the election of active server, this method will block until the server is active. + */ + boolean participateElection(); + + /** + * Add a listener to listen to the status change of the server. + * + * @param listener listener to add. + */ + void addServerStatusChangeListener(ServerStatusChangeListener listener); + + /** + * Get the status of the server. + * + * @return the status of the server. + */ + ServerStatus getServerStatus(); + + /** + * Shutdown the server, release resources. + */ + void shutdown(); + + enum ServerStatus { + ACTIVE, + STAND_BY, + ; + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java new file mode 100644 index 0000000000..af109228e2 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +public interface ServerStatusChangeListener { + + void change(HAServer.ServerStatus originStatus, HAServer.ServerStatus currentStatus); + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 6833a6607b..24a462c03f 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -34,6 +34,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.net.ssl.SSLException; @@ -311,6 +313,35 @@ public class EtcdRegistry implements Registry { } } + @Override + public boolean acquireLock(String key, long timeout) { + Lock lockClient = client.getLockClient(); + Lease leaseClient = client.getLeaseClient(); + // get the lock with a lease + try { + long leaseId = leaseClient.grant(TIME_TO_LIVE_SECONDS).get().getID(); + // keep the lease + lockClient.lock(byteSequence(key), leaseId).get(timeout, TimeUnit.MICROSECONDS); + client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { + })); + + // save the leaseId for release Lock + if (null == threadLocalLockMap.get()) { + threadLocalLockMap.set(new HashMap<>()); + } + threadLocalLockMap.get().put(key, leaseId); + return true; + } catch (TimeoutException timeoutException) { + log.debug("Acquire lock: {} in {}/ms timeout", key, timeout); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RegistryException("etcd get lock error", e); + } catch (ExecutionException e) { + throw new RegistryException("etcd get lock error, lockKey: " + key, e); + } + } + /** * release the lock by revoking the leaseId */ diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java index 70593e4bad..eb716378d0 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java @@ -36,6 +36,7 @@ class EtcdKeepAliveLeaseManagerTest { static Client client; static EtcdKeepAliveLeaseManager etcdKeepAliveLeaseManager; + @BeforeAll public static void before() throws Exception { server = EtcdClusterExtension.builder() @@ -65,8 +66,9 @@ class EtcdKeepAliveLeaseManagerTest { @AfterAll public static void after() throws IOException { - try (EtcdCluster closeServer = server.cluster()) { - client.close(); + try ( + EtcdCluster closeServer = server.cluster(); + Client closedClient = client) { } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java index f3cbcfbc3b..12b29c34cc 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java @@ -179,6 +179,17 @@ public class JdbcRegistry implements Registry { } } + @Override + public boolean acquireLock(String key, long timeout) { + try { + return registryLockManager.acquireLock(key, timeout); + } catch (RegistryException e) { + throw e; + } catch (Exception e) { + throw new RegistryException(String.format("Acquire lock: %s error", key), e); + } + } + @Override public boolean releaseLock(String key) { registryLockManager.releaseLock(key); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java index 46ccd15ec0..b624b9e788 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java @@ -83,6 +83,30 @@ public class RegistryLockManager implements AutoCloseable { }); } + /** + * Acquire the lock, if cannot get the lock will await. + */ + public boolean acquireLock(String lockKey, long timeout) throws RegistryException { + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeout) { + try { + if (lockHoldMap.containsKey(lockKey)) { + return true; + } + JdbcRegistryLock jdbcRegistryLock = jdbcOperator.tryToAcquireLock(lockKey); + if (jdbcRegistryLock != null) { + lockHoldMap.put(lockKey, jdbcRegistryLock); + return true; + } + } catch (SQLException e) { + throw new RegistryException("Acquire the lock: " + lockKey + " error", e); + } + log.debug("Acquire the lock {} failed try again", lockKey); + ThreadUtils.sleep(JdbcRegistryConstant.LOCK_ACQUIRE_INTERVAL); + } + return false; + } + public void releaseLock(String lockKey) { JdbcRegistryLock jdbcRegistryLock = lockHoldMap.get(lockKey); if (jdbcRegistryLock != null) { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 3f0c3ccb59..38c211dfe7 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -217,11 +217,41 @@ public final class ZookeeperRegistry implements Registry { public boolean acquireLock(String key) { InterProcessMutex interProcessMutex = new InterProcessMutex(client, key); try { - interProcessMutex.acquire(); - if (null == threadLocalLockMap.get()) { - threadLocalLockMap.set(new HashMap<>(3)); + if (interProcessMutex.isAcquiredInThisProcess()) { + return true; } - threadLocalLockMap.get().put(key, interProcessMutex); + Map processMutexMap = threadLocalLockMap.get(); + if (null == processMutexMap) { + processMutexMap = new HashMap<>(); + threadLocalLockMap.set(processMutexMap); + } + interProcessMutex.acquire(); + processMutexMap.put(key, interProcessMutex); + return true; + } catch (Exception e) { + try { + interProcessMutex.release(); + throw new RegistryException(String.format("zookeeper get lock: %s error", key), e); + } catch (Exception exception) { + throw new RegistryException(String.format("zookeeper get lock: %s error", key), e); + } + } + } + + @Override + public boolean acquireLock(String key, long timeout) { + InterProcessMutex interProcessMutex = new InterProcessMutex(client, key); + try { + if (interProcessMutex.isAcquiredInThisProcess()) { + return true; + } + Map processMutexMap = threadLocalLockMap.get(); + if (null == processMutexMap) { + processMutexMap = new HashMap<>(); + threadLocalLockMap.set(processMutexMap); + } + interProcessMutex.acquire(timeout, MILLISECONDS); + processMutexMap.put(key, interProcessMutex); return true; } catch (Exception e) { try { @@ -235,13 +265,17 @@ public final class ZookeeperRegistry implements Registry { @Override public boolean releaseLock(String key) { - if (null == threadLocalLockMap.get().get(key)) { + Map processMutexMap = threadLocalLockMap.get(); + if (processMutexMap == null) { + return true; + } + if (null == processMutexMap.get(key)) { return false; } try { - threadLocalLockMap.get().get(key).release(); - threadLocalLockMap.get().remove(key); - if (threadLocalLockMap.get().isEmpty()) { + processMutexMap.get(key).release(); + processMutexMap.remove(key); + if (processMutexMap.isEmpty()) { threadLocalLockMap.remove(); } } catch (Exception e) { diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 6757718929..1c6da324ee 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -232,7 +232,8 @@ alert: # Define value is (0 = infinite), and alert server would be waiting alert result. wait-timeout: 0 max-heartbeat-interval: 60s - query_alert_threshold: 100 + # The maximum number of alerts that can be processed in parallel + sender-parallelism: 5 api: audit-enable: false From ba5de75829f63f0816c9bf4fee7f7e1af4db6fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E9=98=B3=E9=98=B3=28liuyangyang=29?= <1024717602@qq.com> Date: Thu, 9 May 2024 13:06:22 +0800 Subject: [PATCH 86/88] Add tenantCode propagation to DynamicCommandUtils.createCommand (#15956) --- .../server/master/runner/task/dynamic/DynamicCommandUtils.java | 1 + .../master/runner/task/dynamic/DynamicCommandUtilsTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java index e360a8857e..2401562f15 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java @@ -65,6 +65,7 @@ public class DynamicCommandUtils { command.setProcessInstancePriority(processInstance.getProcessInstancePriority()); command.setWorkerGroup(processInstance.getWorkerGroup()); command.setDryRun(processInstance.getDryRun()); + command.setTenantCode(processInstance.getTenantCode()); return command; } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java index d238869f41..d9b9c82e66 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java @@ -54,6 +54,7 @@ class DynamicCommandUtilsTest { processInstance.setWarningGroupId(1); processInstance.setProcessInstancePriority(null); // update this processInstance.setWorkerGroup("worker"); + processInstance.setTenantCode("unit-root"); processInstance.setDryRun(0); } @@ -73,6 +74,7 @@ class DynamicCommandUtilsTest { Assertions.assertEquals(processInstance.getProcessInstancePriority(), command.getProcessInstancePriority()); Assertions.assertEquals(processInstance.getWorkerGroup(), command.getWorkerGroup()); Assertions.assertEquals(processInstance.getDryRun(), command.getDryRun()); + Assertions.assertEquals(processInstance.getTenantCode(), command.getTenantCode()); } @Test From 5c569b705cad84be0b017dc94dfdd979318e3279 Mon Sep 17 00:00:00 2001 From: Zzih96 <158246610+Zzih96@users.noreply.github.com> Date: Thu, 9 May 2024 14:15:06 +0800 Subject: [PATCH 87/88] [fix-15907] Fix get remote shell exit code is incorrect (#15911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ./mvnw spotless:apply * Update dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java Co-authored-by: Wenjun Ruan --------- Co-authored-by: 詹子恒 Co-authored-by: Wenjun Ruan Co-authored-by: Rick Cheng --- .../plugin/task/remoteshell/RemoteExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java index 307023043a..814826a6bb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteExecutor.java @@ -131,7 +131,7 @@ public class RemoteExecutor implements AutoCloseable { int exitCode = -1; log.info("Remote shell task run status: {}", logLine); if (logLine.contains(STATUS_TAG_MESSAGE)) { - String status = logLine.replace(STATUS_TAG_MESSAGE, "").trim(); + String status = StringUtils.substringAfter(logLine, STATUS_TAG_MESSAGE); if (status.equals("0")) { log.info("Remote shell task success"); exitCode = 0; From 60b019b729a5bb1c05e5627d85b1a903546100a8 Mon Sep 17 00:00:00 2001 From: cntiger <35484811+cntigers@users.noreply.github.com> Date: Thu, 9 May 2024 14:50:27 +0800 Subject: [PATCH 88/88] [Improvement] Fix the git url command injection in pytorch task(#15873) (#15950) * fix the git url command injection danger(#15873) * [Improvement] Fix the git url command injection in pytorch,format code style task(#15873) --------- Co-authored-by: cntigers Co-authored-by: Rick Cheng --- .../plugin/task/pytorch/GitProjectManager.java | 4 ++-- .../plugin/task/pytorch/PytorchTaskTest.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java index 3189f26920..5f1e815c30 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java @@ -33,12 +33,12 @@ import lombok.extern.slf4j.Slf4j; public class GitProjectManager { public static final String GIT_PATH_LOCAL = "GIT_PROJECT"; - private static final Pattern GIT_CHECK_PATTERN = Pattern.compile("^(git@|https?://)"); + private static final Pattern GIT_CHECK_PATTERN = Pattern.compile("^(git@|https?://)(?![&|])[^&|]+$"); private String path; private String baseDir = "."; public static boolean isGitPath(String path) { - return GIT_CHECK_PATTERN.matcher(path).find(); + return GIT_CHECK_PATTERN.matcher(path).matches(); } public void prepareProject() throws Exception { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java index c213021607..e35a175df1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java @@ -72,6 +72,12 @@ public class PytorchTaskTest { } + @Test + public void testGitProjectUrlInjection() { + Assertions.assertFalse(GitProjectManager.isGitPath("git@& cat /etc/passwd >/poc.txt #")); + Assertions.assertFalse(GitProjectManager.isGitPath("git@| cat /etc/passwd >/poc.txt #")); + } + @Test public void testGitProject() {