From e6245b099991222dee390308bb4ff1add434d619 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 21:39:36 +0800 Subject: [PATCH 01/16] remove registry pure --- .../dolphinscheduler-alert-server/pom.xml | 24 ------------- dolphinscheduler-api/pom.xml | 8 ----- dolphinscheduler-master/pom.xml | 8 ----- .../dolphinscheduler-registry-pure/pom.xml | 36 ------------------- dolphinscheduler-registry/pom.xml | 1 - dolphinscheduler-worker/pom.xml | 8 ----- pom.xml | 5 --- 7 files changed, 90 deletions(-) delete mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-pure/pom.xml diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 808f7123c7..13483bddb3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -115,30 +115,6 @@ - - task-pure - - false - - - - org.apache.dolphinscheduler - dolphinscheduler-registry-pure - - - - - task-all - - true - - - - org.apache.dolphinscheduler - dolphinscheduler-registry-all - - - diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 0bcd4499d8..be7bbe8ee8 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -294,10 +294,6 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-pure - - org.apache.dolphinscheduler - dolphinscheduler-registry-pure - org.apache.dolphinscheduler dolphinscheduler-storage-pure @@ -318,10 +314,6 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-all - - org.apache.dolphinscheduler - dolphinscheduler-registry-all - org.apache.dolphinscheduler dolphinscheduler-storage-all diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 65191b3557..6c45e2150f 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -341,10 +341,6 @@ - - org.apache.dolphinscheduler - dolphinscheduler-registry-pure - org.apache.dolphinscheduler dolphinscheduler-storage-pure @@ -361,10 +357,6 @@ org.apache.dolphinscheduler dolphinscheduler-task-all - - org.apache.dolphinscheduler - dolphinscheduler-registry-all - org.apache.dolphinscheduler dolphinscheduler-storage-all diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-pure/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-pure/pom.xml deleted file mode 100644 index 5f147418a6..0000000000 --- a/dolphinscheduler-registry/dolphinscheduler-registry-pure/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - org.apache.dolphinscheduler - dolphinscheduler-registry - 3.2.3-SNAPSHOT - - - dolphinscheduler-registry-pure - - - - org.apache.dolphinscheduler - dolphinscheduler-registry-zookeeper - ${project.version} - - - diff --git a/dolphinscheduler-registry/pom.xml b/dolphinscheduler-registry/pom.xml index fd8e3d842c..e74cde5cbb 100644 --- a/dolphinscheduler-registry/pom.xml +++ b/dolphinscheduler-registry/pom.xml @@ -30,7 +30,6 @@ dolphinscheduler-registry-api dolphinscheduler-registry-plugins dolphinscheduler-registry-all - dolphinscheduler-registry-pure diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index bfec21b5d8..99177b28fd 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -167,10 +167,6 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-pure - - org.apache.dolphinscheduler - dolphinscheduler-registry-pure - org.apache.dolphinscheduler dolphinscheduler-storage-pure @@ -191,10 +187,6 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-all - - org.apache.dolphinscheduler - dolphinscheduler-registry-all - org.apache.dolphinscheduler dolphinscheduler-storage-all diff --git a/pom.xml b/pom.xml index 85d990807f..a4756e1fee 100755 --- a/pom.xml +++ b/pom.xml @@ -199,11 +199,6 @@ dolphinscheduler-registry-all ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-registry-pure - ${project.version} - org.apache.dolphinscheduler From 19a73c8630a537c3edfb6697835ee00cc2dc9dae Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 22:02:15 +0800 Subject: [PATCH 02/16] =?UTF-8?q?add=20registry-all=20to=20alert=E3=80=81a?= =?UTF-8?q?pi=E3=80=81master=E3=80=81worker=20pom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dolphinscheduler-alert-server/pom.xml | 5 +++++ dolphinscheduler-api/pom.xml | 6 ++++++ dolphinscheduler-master/pom.xml | 5 +++++ dolphinscheduler-worker/pom.xml | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 13483bddb3..343677a256 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -43,6 +43,11 @@ dolphinscheduler-dao + + org.apache.dolphinscheduler + dolphinscheduler-registry-all + + org.apache.dolphinscheduler dolphinscheduler-extract-alert diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index be7bbe8ee8..290b3e0bb3 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -70,6 +70,12 @@ org.apache.dolphinscheduler dolphinscheduler-ui + + + org.apache.dolphinscheduler + dolphinscheduler-registry-all + + org.apache.dolphinscheduler dolphinscheduler-scheduler-all diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 6c45e2150f..53f8753dc5 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -55,6 +55,11 @@ dolphinscheduler-service + + org.apache.dolphinscheduler + dolphinscheduler-registry-all + + org.apache.dolphinscheduler dolphinscheduler-scheduler-all diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index 99177b28fd..15057a9cbc 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -57,6 +57,11 @@ dolphinscheduler-extract-alert + + org.apache.dolphinscheduler + dolphinscheduler-registry-all + + org.apache.dolphinscheduler dolphinscheduler-extract-master From 522b19b0af724ade7ec8aaba92c9725dfb26bee8 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 22:14:29 +0800 Subject: [PATCH 03/16] mvn spotless:apply --- dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 343677a256..53aa30814b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -47,7 +47,7 @@ org.apache.dolphinscheduler dolphinscheduler-registry-all - + org.apache.dolphinscheduler dolphinscheduler-extract-alert From c267cdf72a3b4871b242d7894d1946f080c394b1 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 22:40:14 +0800 Subject: [PATCH 04/16] try to fix API-Test --- dolphinscheduler-api/pom.xml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 290b3e0bb3..30915ee70f 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -91,6 +91,11 @@ dolphinscheduler-extract-worker + + org.apache.dolphinscheduler + dolphinscheduler-datasource-all + + org.codehaus.janino janino @@ -296,10 +301,6 @@ org.apache.dolphinscheduler dolphinscheduler-task-pure - - org.apache.dolphinscheduler - dolphinscheduler-datasource-pure - org.apache.dolphinscheduler dolphinscheduler-storage-pure @@ -316,10 +317,6 @@ org.apache.dolphinscheduler dolphinscheduler-task-all - - org.apache.dolphinscheduler - dolphinscheduler-datasource-all - org.apache.dolphinscheduler dolphinscheduler-storage-all From 93d4d2a184d5f22f14ae58c44d5dbe30bff46791 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 22:43:49 +0800 Subject: [PATCH 05/16] mvn spotless:apply --- dolphinscheduler-api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 30915ee70f..0fd6ceb451 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -95,7 +95,7 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-all - + org.codehaus.janino janino From d5df5ccda51777f5c18c0ef1caef6efecadcee1c Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 22:54:11 +0800 Subject: [PATCH 06/16] try to fix worker E2E-K8S-EXECUTE --- dolphinscheduler-master/pom.xml | 10 ++++++++++ dolphinscheduler-worker/pom.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 53f8753dc5..3c16aaee6c 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -75,6 +75,16 @@ dolphinscheduler-extract-worker + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + + + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + org.springframework.boot spring-boot-starter-cache diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index 15057a9cbc..124ecf3c5c 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -72,6 +72,16 @@ dolphinscheduler-extract-worker + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + + + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + org.springframework.boot spring-boot-starter From 063d0ad81d0468b3908938ba84765b1cb1749daa Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 23:23:42 +0800 Subject: [PATCH 07/16] move sparkParameters from spark to api --- .../server/master/runner/TaskExecutionContextFactory.java | 2 +- .../plugin/task/api/parameters}/SparkParameters.java | 4 ++-- .../apache/dolphinscheduler/plugin/task/spark/SparkTask.java | 2 ++ .../dolphinscheduler/plugin/task/spark/SparkTaskChannel.java | 1 + .../plugin/task/spark/SparkParametersTest.java | 1 + .../dolphinscheduler/plugin/task/spark/SparkTaskTest.java | 2 ++ 6 files changed, 9 insertions(+), 3 deletions(-) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark => dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters}/SparkParameters.java (95%) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index b3b0cf67ea..e9cf333b1e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -57,7 +57,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.UdfFuncParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.JdbcUrlParser; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; -import org.apache.dolphinscheduler.plugin.task.spark.SparkParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.exception.TaskExecutionContextCreateException; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SparkParameters.java similarity index 95% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SparkParameters.java index 873ba22c71..0eae7ab3b4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SparkParameters.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.spark; +package org.apache.dolphinscheduler.plugin.task.api.parameters; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index 3c5fc17698..a1bb70beb4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -25,12 +25,14 @@ import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.SPARK import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.SPARK_ON_K8S_MASTER_PREFIX; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java index 58211ea1ac..3cfb72fd0a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class SparkTaskChannel implements TaskChannel { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java index 19ec707c62..c9c7375c44 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import java.util.LinkedList; import java.util.List; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index 5138562564..28d3fbb5af 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -21,8 +21,10 @@ import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.TYPE_ import static org.mockito.ArgumentMatchers.any; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.commons.io.FileUtils; From dff98f0492640937dbafb050600ab385bccfa34d Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 8 Jun 2024 23:24:51 +0800 Subject: [PATCH 08/16] mvn spotless:apply --- .../server/master/runner/TaskExecutionContextFactory.java | 2 +- .../apache/dolphinscheduler/plugin/task/spark/SparkTask.java | 2 +- .../dolphinscheduler/plugin/task/spark/SparkParametersTest.java | 2 +- .../dolphinscheduler/plugin/task/spark/SparkTaskTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index e9cf333b1e..3226f6e0a0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -50,6 +50,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.AbstractResourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; @@ -57,7 +58,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.UdfFuncParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.JdbcUrlParser; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.exception.TaskExecutionContextCreateException; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index a1bb70beb4..ab143df4a4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -24,8 +24,8 @@ import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.DRIVE import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.SPARK_KUBERNETES_NAMESPACE; import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.SPARK_ON_K8S_MASTER_PREFIX; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.enums.ProgramType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java index c9c7375c44..457842c132 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import java.util.LinkedList; import java.util.List; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index 28d3fbb5af..f3f3ef55ff 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.plugin.task.spark; import static org.apache.dolphinscheduler.plugin.task.spark.SparkConstants.TYPE_FILE; import static org.mockito.ArgumentMatchers.any; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.enums.ProgramType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.SparkParameters; From 0ae0b0155f7248fc5570b1559f9240480ca5abc7 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 08:57:50 +0800 Subject: [PATCH 09/16] try to fix k8s start error --- deploy/kubernetes/dolphinscheduler/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 52addbac19..9c8d7c6f83 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -49,6 +49,9 @@ image: # -- tools image tools: dolphinscheduler-tools +datasource: + profile: postgresql + postgresql: # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL enabled: true From a381894b1ddbc73a4cad8089206d3802179809ef Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 11:32:58 +0800 Subject: [PATCH 10/16] Revert "try to fix k8s start error" This reverts commit 0ae0b0155f7248fc5570b1559f9240480ca5abc7. --- deploy/kubernetes/dolphinscheduler/values.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 9c8d7c6f83..52addbac19 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -49,9 +49,6 @@ image: # -- tools image tools: dolphinscheduler-tools -datasource: - profile: postgresql - postgresql: # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL enabled: true From 29c13fc3c274025007a4f5397d6e44e8af654568 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 12:18:32 +0800 Subject: [PATCH 11/16] delete nimbus-jose-jwt-9.22.jar --- dolphinscheduler-dist/release-docs/LICENSE | 1 - tools/dependencies/known-dependencies.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 6c68096dcd..e01d5ab085 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -528,7 +528,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. reactor-netty-core 1.0.22: https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.0.22, Apache 2.0 reactor-netty-http 1.0.22: https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-http/1.0.22, Apache 2.0 DmJdbcDriver18 8.1.2.79: https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18/8.1.2.79, Apache 2.0 - nimbus-jose-jwt 9.22: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.22, Apache 2.0 woodstox-core 6.4.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/6.4.0, Apache 2.0 auto-value 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value/1.10.1, Apache 2.0 auto-value-annotations 1.10.4: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.4, Apache 2.0 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index d21401974e..4a591f8a08 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -431,7 +431,6 @@ azure-security-keyvault-keys-4.5.4.jar azure-security-keyvault-secrets-4.5.4.jar azure-storage-common-12.20.0.jar azure-storage-file-share-12.17.0.jar -nimbus-jose-jwt-9.22.jar woodstox-core-6.4.0.jar azure-core-management-1.10.1.jar api-common-2.6.0.jar From 00f551dd8d04143f3a8aec397b00975442355035 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 12:36:14 +0800 Subject: [PATCH 12/16] modify dependency version of dolphinscheduler-api-test-case --- .../dolphinscheduler-api-test-case/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml index 5b16b3388d..d5f3436740 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml @@ -39,12 +39,12 @@ org.apache.dolphinscheduler dolphinscheduler-dao - dev-SNAPSHOT + ${project.version} org.apache.dolphinscheduler dolphinscheduler-api - dev-SNAPSHOT + ${project.version} From db4c789ff26ac7e3a5281547561a969745bd685b Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 13:05:47 +0800 Subject: [PATCH 13/16] Revert "modify dependency version of dolphinscheduler-api-test-case" This reverts commit 00f551dd8d04143f3a8aec397b00975442355035. --- .../dolphinscheduler-api-test-case/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml index d5f3436740..5b16b3388d 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml @@ -39,12 +39,12 @@ org.apache.dolphinscheduler dolphinscheduler-dao - ${project.version} + dev-SNAPSHOT org.apache.dolphinscheduler dolphinscheduler-api - ${project.version} + dev-SNAPSHOT From 2a811b2e8d98002a6971c9aea27cf6ea0a3c9017 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 13:29:38 +0800 Subject: [PATCH 14/16] modify version of apitest dependency --- .../dolphinscheduler-api-test-case/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml index 5b16b3388d..1cdc059616 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml @@ -39,12 +39,12 @@ org.apache.dolphinscheduler dolphinscheduler-dao - dev-SNAPSHOT + 3.2.3-SNAPSHOT org.apache.dolphinscheduler dolphinscheduler-api - dev-SNAPSHOT + 3.2.3-SNAPSHOT From 6f422b5cbdd1f5b6699058e1b0e6456ffc6c869e Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sun, 9 Jun 2024 21:10:16 +0800 Subject: [PATCH 15/16] add dolphinscheduler-storage-api to api/master/worker --- dolphinscheduler-api/pom.xml | 5 +++++ dolphinscheduler-master/pom.xml | 5 +++++ dolphinscheduler-worker/pom.xml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 0fd6ceb451..e5e0241c82 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -86,6 +86,11 @@ dolphinscheduler-extract-master + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + org.apache.dolphinscheduler dolphinscheduler-extract-worker diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 3c16aaee6c..7645e945af 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -65,6 +65,11 @@ dolphinscheduler-scheduler-all + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + org.apache.dolphinscheduler dolphinscheduler-extract-master diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index 124ecf3c5c..ffcf3134f3 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -52,6 +52,11 @@ dolphinscheduler-meter + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + org.apache.dolphinscheduler dolphinscheduler-extract-alert From 6d2cfdc9a8fccbdb0d6301674bcc252ec323b954 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 10 Jun 2024 08:15:23 +0800 Subject: [PATCH 16/16] try to fix ut --- dolphinscheduler-api/pom.xml | 2 +- dolphinscheduler-master/pom.xml | 4 ++-- dolphinscheduler-worker/pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index e5e0241c82..95275b9933 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -88,7 +88,7 @@ org.apache.dolphinscheduler - dolphinscheduler-storage-api + dolphinscheduler-storage-all diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 7645e945af..ee3a843f1e 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -67,7 +67,7 @@ org.apache.dolphinscheduler - dolphinscheduler-storage-api + dolphinscheduler-storage-all @@ -87,7 +87,7 @@ org.apache.dolphinscheduler - dolphinscheduler-storage-api + dolphinscheduler-storage-all diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index ffcf3134f3..311d78861e 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -54,7 +54,7 @@ org.apache.dolphinscheduler - dolphinscheduler-storage-api + dolphinscheduler-storage-all