diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 507d7acb45..16dbe8be5e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -41,6 +41,12 @@ org.apache.dolphinscheduler dolphinscheduler-dao + + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + + diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml index 5b16b3388d..4d862a6f04 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml @@ -46,5 +46,15 @@ dolphinscheduler-api dev-SNAPSHOT + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.httpcomponents + httpmime + 4.5.13 + diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 681a85318a..9abe058448 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -69,7 +69,7 @@ org.apache.dolphinscheduler - dolphinscheduler-task-all + dolphinscheduler-task-all-prune diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index 88cd3247ae..33a3d3b280 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -66,7 +66,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.facebook.presto.jdbc.internal.guava.base.Strings; /** * worker group service impl @@ -247,7 +246,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro * @return boolean */ private String checkWorkerGroupAddrList(WorkerGroup workerGroup) { - if (Strings.isNullOrEmpty(workerGroup.getAddrList())) { + if (StringUtils.isEmpty(workerGroup.getAddrList())) { return null; } Map serverMaps = registryClient.getServerMaps(RegistryNodeType.WORKER); diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index 6dd55fca46..e1152da947 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -67,6 +67,23 @@ ${basedir}/../dolphinscheduler-tools/target/tools tools + + libs/ + bin/ + dist-bin/ + + + + ${basedir}/../dolphinscheduler-tools/target/tools/libs + tools/libs + + dolphinscheduler-*.jar + spring-*.jar + + + + ${basedir}/../dolphinscheduler-tools/target/tools/dist-bin + tools/bin @@ -74,11 +91,6 @@ . - - ${basedir}/../dolphinscheduler-ui/dist - ./ui - - ${basedir}/../script bin diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 31627d4b7f..e5d77dc324 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -66,12 +66,7 @@ org.apache.dolphinscheduler - dolphinscheduler-task-all - - - - org.apache.dolphinscheduler - dolphinscheduler-storage-all + dolphinscheduler-task-all-prune diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java index 9b901d7b60..ed56c34501 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java @@ -24,7 +24,7 @@ import java.util.function.Supplier; import lombok.experimental.UtilityClass; -import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; +import com.google.common.collect.ImmutableSet; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml index 90fa9cb5e0..fb422e2a48 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml @@ -38,5 +38,10 @@ dolphinscheduler-task-api ${project.version} + + + com.aliyun.oss + aliyun-sdk-oss + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml new file mode 100644 index 0000000000..eb251c3fb9 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml @@ -0,0 +1,425 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-task-plugin + dev-SNAPSHOT + + + dolphinscheduler-task-all-prune + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-dataquality + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-datax + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-flink + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-flink-stream + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-http + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-mr + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-pigeon + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-procedure + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-python + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-seatunnel + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-shell + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-spark + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-sql + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-sqoop + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-emr + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-zeppelin + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-jupyter + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-k8s + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-mlflow + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-openmldb + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-dvc + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-dinky + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-java + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-sagemaker + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-pytorch + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-hivecli + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-chunjun + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-dms + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-datasync + ${project.version} + + + * + * + + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-kubeflow + ${project.version} + + + * + * + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-linkis + ${project.version} + + + * + * + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-datafactory + ${project.version} + + + * + * + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-remoteshell + ${project.version} + + + * + * + + + + + + diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index f4c1573226..ec7635e4d5 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -29,6 +29,7 @@ dolphinscheduler-task-all + dolphinscheduler-task-all-prune dolphinscheduler-task-api dolphinscheduler-task-shell dolphinscheduler-task-datax diff --git a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml index 49a95d25d0..7b206da233 100644 --- a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml +++ b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml @@ -38,6 +38,12 @@ 0755 0755 + + ${basedir}/src/main/dist-bin + dist-bin + 0755 + 0755 + ${basedir}/../dolphinscheduler-dao/src/main/resources diff --git a/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh b/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh new file mode 100644 index 0000000000..0be4cd3a11 --- /dev/null +++ b/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt +CP=$DOLPHINSCHEDULER_HOME/tools/libs/* +for d in api-server; do + for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do + JAR_FILE_NAME=${f##*/} + if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then + CP=$CP:$f + fi + done +done + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$CP":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=demo,${DATABASE} \ + org.apache.dolphinscheduler.tools.demo.CreateProcessDemo diff --git a/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh b/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh new file mode 100644 index 0000000000..d4422d763e --- /dev/null +++ b/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt +CP=$DOLPHINSCHEDULER_HOME/tools/libs/* +for d in api-server; do + for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do + JAR_FILE_NAME=${f##*/} + if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then + CP=$CP:$f + fi + done +done + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$CP":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=resource,${DATABASE} \ + org.apache.dolphinscheduler.tools.resource.MigrateResource $1 diff --git a/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh b/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh new file mode 100755 index 0000000000..4f768ca7b2 --- /dev/null +++ b/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt +CP=$DOLPHINSCHEDULER_HOME/tools/libs/* +for d in api-server; do + for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do + JAR_FILE_NAME=${f##*/} + if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then + CP=$CP:$f + fi + done +done + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/sql":"$CP" \ + -Dspring.profiles.active=upgrade,${DATABASE} \ + org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler diff --git a/pom.xml b/pom.xml index 260524b1e5..887162b8ac 100755 --- a/pom.xml +++ b/pom.xml @@ -232,6 +232,11 @@ dolphinscheduler-task-all ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-task-all-prune + ${project.version} + org.apache.dolphinscheduler diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 64cef29113..ce7e6565ea 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -11,7 +11,6 @@ annotations-13.0.jar apache-client-2.17.282.jar asm-9.1.jar aspectjweaver-1.9.7.jar -aspectjrt-1.9.7.jar auth-2.17.282.jar audience-annotations-0.12.0.jar avro-1.7.7.jar @@ -23,9 +22,7 @@ aws-java-sdk-s3-1.12.300.jar aws-java-sdk-sagemaker-1.12.300.jar aws-java-sdk-dms-1.12.300.jar aws-json-protocol-2.17.282.jar -bcpkix-jdk15on-1.69.jar bcprov-ext-jdk15on-1.69.jar -bcprov-jdk15on-1.69.jar bcutil-jdk15on-1.69.jar bonecp-0.8.0.RELEASE.jar bucket4j-core-6.2.0.jar @@ -435,8 +432,8 @@ azure-core-management-1.10.1.jar api-common-2.6.0.jar auto-value-1.10.1.jar auto-value-annotations-1.10.1.jar -bcpkix-jdk15on-1.67.jar -bcprov-jdk15on-1.67.jar +bcpkix-jdk15on-1.69.jar +bcprov-jdk15on-1.69.jar conscrypt-openjdk-uber-2.5.2.jar gapic-google-cloud-storage-v2-2.18.0-alpha.jar gax-2.23.0.jar