Add TaskExecutionRunnable in WorkflowExecutionDAG

.asf.yaml

@@ -46,7 +46,7 @@ github:
           - E2E
           - Docs
           - Frontend Build
-#          - "Mergeable: milestone-label-check"
+          - "Mergeable: milestone-label-check"
       required_pull_request_reviews:
         dismiss_stale_reviews: true
-        required_approving_review_count: 2
+        required_approving_review_count: 1

.github/ISSUE_TEMPLATE/dsip-request.yml

@@ -1,77 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-name: DSIP
-description: Suggest an idea for this project
-title: "[DSIP-][Module Name] DSIP title"
-labels: [ "DSIP", "Waiting for reply" ]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        For better global communication, Please write in English.
-
-        If you feel the description in English is not clear, then you can append description in Chinese, thanks!
-
-  - type: checkboxes
-    attributes:
-      label: Search before asking
-      description: >
-        Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first
-        to see whether the same DSIP was created already.
-      options:
-        - label: >
-            I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no
-            similar DSIP.
-          required: true
-
-  - type: textarea
-    attributes:
-      label: Motivation
-      description: Why you want to do this change?
-
-  - type: textarea
-    attributes:
-      label: Design Detail
-      description: Your design.
-      placeholder: >
-        It's better to provide a detailed design, such as the design of the interface, the design of the database, etc.
-
-  - type: textarea
-    attributes:
-      label: Compatibility, Deprecation, and Migration Plan
-      description: >
-        If this feature is related to compatibility, deprecation, or migration, please describe it here.
-
-  - type: textarea
-    attributes:
-      label: Test Plan
-      description: >
-        How to test this improvement.
-
-  - type: checkboxes
-    attributes:
-      label: Code of Conduct
-      description: |
-        The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
-      options:
-        - label: |
-            I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
-          required: true
-
-  - type: markdown
-    attributes:
-      value: "Thanks for completing our form!"

.github/mergeable.yml

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: 2
+mergeable:
+  # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
+  # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
+  # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
+  # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
+  - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
+    name: sync-sql-ddl
+    validate:
+      # Sql files must change synchronize
+      - do: dependent
+        files:
+          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
+          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
+          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
+        message: 'Sql files not change synchronize'
+    # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
+    fail:
+      - do: comment
+        payload:
+          body: >
+            :warning: This PR do not change database DDL synchronize.
+        leave_old_comment: false
+      - do: labels
+        add: 'sql not sync'
+    # Remove labels 'sql not sync' if pass
+    pass:
+      - do: labels
+        delete: 'sql not sync'
+
+  - when: pull_request.*
+    name: milestone-label-check
+    validate:
+      - do: milestone
+        no_empty:
+          enabled: false # Cannot be empty when true.
+          message: 'Milestone is required and cannot be empty.'
+      - do: label
+        and:
+          - must_include:
+              regex: 'feature|bug|improvement|document|chore|revert'
+              message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
+          - must_include:
+              regex: 'ready-to-merge'
+              message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'

.github/workflows/backend.yml

@@ -106,14 +106,10 @@ jobs:
     strategy:
       matrix:
         case:
-          - name: cluster-test-mysql-with-zookeeper-registry
+          - name: cluster-test-mysql
-            script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
+            script: .github/workflows/cluster-test/mysql/start-job.sh
-          - name: cluster-test-mysql-with-mysql-registry
+          - name: cluster-test-postgresql
-            script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
+            script: .github/workflows/cluster-test/postgresql/start-job.sh
-          - name: cluster-test-postgresql-zookeeper-registry
-            script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
-          - name: cluster-test-postgresql-with-postgresql-registry
-            script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
     steps:
       - uses: actions/checkout@v2
         with:
@@ -153,7 +149,7 @@ jobs:
       fail-fast: false
       matrix:
         db: ["mysql", "postgresql"]
-        version: ["3.1.9", "3.2.0"]
+        version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"]
     steps:
       - name: Set up JDK 8
         uses: actions/setup-java@v2

.github/workflows/cluster-test/mysql/Dockerfile

@@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
 ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting install.sh
-COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
 
 #Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Download mysql jar
 ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
@@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME
 cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
 
 #Deploy
-COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh
+COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
 
 CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh

@@ -28,6 +28,7 @@ export SPRING_DATASOURCE_PASSWORD=123456
 # DolphinScheduler server related configuration
 export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
 export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
 
 # Registry center configuration, determines the type and link of the registry center
 export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}

.github/workflows/cluster-test/mysql/start-job.sh

@@ -18,16 +18,16 @@
 set -euox pipefail
 
 #Start base service containers
-docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d
+docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
 
 #Build ds mysql cluster image
-docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile .
+docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile .
 
 #Start ds mysql cluster container
-docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d
+docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
 
 #Running tests
-/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
+/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
 
 #Cleanup
 docker rm -f $(docker ps -aq)

.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -euox pipefail
-
-
-USER=root
-
-#Create database
-mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
-
-#Sudo
-sed -i '$a'$USER'  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
-sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
-
-#SSH
-ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-chmod 600 ~/.ssh/authorized_keys
-service ssh start
-
-#Init schema
-/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
-/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
-
-#Start Cluster
-/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
-
-#Keep running
-tail -f /dev/null

.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml

@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: "3"
-
-services:
-  mysql:
-    container_name: mysql
-    image: mysql:5.7.36
-    command: --default-authentication-plugin=mysql_native_password
-    restart: always
-    environment:
-      MYSQL_ROOT_PASSWORD: 123456
-    ports:
-      - "3306:3306"
-    healthcheck:
-      test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
-      interval: 5s
-      timeout: 60s
-      retries: 120

.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh

@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# JAVA_HOME, will use it to start DolphinScheduler server
-export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
-
-# Database related configuration, set database type, username and password
-export DATABASE=${DATABASE:-mysql}
-export SPRING_PROFILES_ACTIVE=${DATABASE}
-export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
-export SPRING_DATASOURCE_USERNAME=root
-export SPRING_DATASOURCE_PASSWORD=123456
-
-# DolphinScheduler server related configuration
-export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
-export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
-
-# Registry center configuration, determines the type and link of the registry center
-export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
-export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
-export REGISTRY_HIKARI_CONFIG_USERNAME=root
-export REGISTRY_HIKARI_CONFIG_PASSWORD=123456
-
-# Tasks related configurations, need to change the configuration if you use the related tasks.
-export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
-export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
-export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
-export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
-export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
-export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
-
-export MASTER_RESERVED_MEMORY=0.01
-export WORKER_RESERVED_MEMORY=0.01
-
-# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
-#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
-#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
-#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
-#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
-#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS

.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh

@@ -1,58 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# ---------------------------------------------------------
-# INSTALL MACHINE
-# ---------------------------------------------------------
-# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
-# including master, worker, api, alert. If you want to deploy in pseudo-distributed
-# mode, just write a pseudo-distributed hostname
-# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
-ips=${ips:-"localhost"}
-
-# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
-# modify it if you use different ssh port
-sshPort=${sshPort:-"22"}
-
-# A comma separated list of machine hostname or IP would be installed Master server, it
-# must be a subset of configuration `ips`.
-# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
-masters=${masters:-"localhost"}
-
-# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
-# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
-# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
-workers=${workers:-"localhost:default"}
-
-# A comma separated list of machine hostname or IP would be installed Alert server, it
-# must be a subset of configuration `ips`.
-# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
-alertServer=${alertServer:-"localhost"}
-
-# A comma separated list of machine hostname or IP would be installed API server, it
-# must be a subset of configuration `ips`.
-# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
-apiServers=${apiServers:-"localhost"}
-
-# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
-# Do not set this configuration same as the current path (pwd)
-installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
-
-# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
-# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
-# to be created by this user
-deployUser=${deployUser:-"dolphinscheduler"}

.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile

@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM eclipse-temurin:8-jre
-
-RUN apt update ; \
-    apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ;
-
-COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
-RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
-
-RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
-
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
-
-#Setting install.sh
-COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
-
-#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
-
-#Download mysql jar
-ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
-ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
-RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
-
-#Deploy
-COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh
-
-CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml

@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: "3"
-
-services:
-  ds:
-    container_name: ds
-    image: jdk8:ds_mysql_cluster
-    restart: always
-    ports:
-      - "12345:12345"
-      - "5679:5679"
-      - "1235:1235"
-      - "50053:50053"

.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh

@@ -1,108 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -x
-
-
-API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
-MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
-WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
-ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
-
-#Cluster start health check
-TIMEOUT=180
-START_HEALTHCHECK_EXITCODE=0
-
-for ((i=1; i<=TIMEOUT; i++))
-do
-  MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
-  WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
-  API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
-  ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
-  if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
-    START_HEALTHCHECK_EXITCODE=0
-  else
-    START_HEALTHCHECK_EXITCODE=2
-  fi
-
-  if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
-    echo "cluster start health check success"
-    break
-  fi
-
-  if [[ $i -eq $TIMEOUT ]];then
-    if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
-      echo "master start health check failed"
-    fi
-    if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
-      echo "worker start health check failed"
-    fi
-    if [[ $API_HTTP_STATUS -ne 200 ]]; then
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
-      echo "api start health check failed"
-    fi
-    if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
-      docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
-      echo "alert start health check failed"
-    fi
-    exit $START_HEALTHCHECK_EXITCODE
-  fi
-  sleep 1
-done
-
-#Stop Cluster
-docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
-
-#Cluster stop health check
-sleep 5
-MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
-if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-  echo "master stop health check success"
-else
-  echo "master stop health check failed"
-  exit 3
-fi
-
-WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
-if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
-  echo "worker stop health check success"
-else
-  echo "worker stop health check failed"
-  exit 3
-fi
-
-API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
-if [[ $API_HTTP_STATUS -ne 200 ]];then
-  echo "api stop health check success"
-else
-  echo "api stop health check failed"
-  exit 3
-fi
-
-ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
-if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
-  echo "alert stop health check success"
-else
-  echo "alert stop health check failed"
-  exit 3
-fi

.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -euox pipefail
-
-#Start base service containers
-docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d
-
-#Build ds mysql cluster image
-docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile .
-
-#Start ds mysql cluster container
-docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
-
-#Running tests
-/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
-
-#Cleanup
-docker rm -f $(docker ps -aq)

.github/workflows/cluster-test/postgresql/Dockerfile

@@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
 ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting install.sh
-COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
 
 #Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Deploy
-COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh
+COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
 
 CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh

@@ -28,6 +28,7 @@ export SPRING_DATASOURCE_PASSWORD=postgres
 # DolphinScheduler server related configuration
 export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
 export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
 
 # Registry center configuration, determines the type and link of the registry center
 export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}

.github/workflows/cluster-test/postgresql/start-job.sh

@@ -18,16 +18,16 @@
 set -euox pipefail
 
 #Start base service containers
-docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d
+docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
 
 #Build ds postgresql cluster image
-docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile .
+docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile .
 
 #Start ds postgresql cluster container
-docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
+docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
 
 #Running tests
-/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
+/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
 
 #Cleanup
 docker rm -f $(docker ps -aq)

.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile

@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM eclipse-temurin:8-jre
-
-RUN apt update ; \
-    apt install -y wget sudo openssh-server netcat-traditional ;
-
-COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
-RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
-
-RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
-
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
-
-#Setting install.sh
-COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
-
-#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
-
-#Deploy
-COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh
-
-CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -euox pipefail
-
-
-USER=root
-
-#Sudo
-sed -i '$a'$USER'  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
-sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
-
-#SSH
-ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-chmod 600 ~/.ssh/authorized_keys
-service ssh start
-
-#Init schema
-/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
-/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
-
-#Start Cluster
-/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
-
-#Keep running
-tail -f /dev/null

.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml

@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: "3"
-
-services:
-  postgres:
-    container_name: postgres
-    image: postgres:14.1
-    restart: always
-    environment:
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: dolphinscheduler
-    ports:
-      - "5432:5432"
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
-

.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh

@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# JAVA_HOME, will use it to start DolphinScheduler server
-export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
-
-# Database related configuration, set database type, username and password
-export DATABASE=${DATABASE:-postgresql}
-export SPRING_PROFILES_ACTIVE=${DATABASE}
-export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
-export SPRING_DATASOURCE_USERNAME=postgres
-export SPRING_DATASOURCE_PASSWORD=postgres
-
-# DolphinScheduler server related configuration
-export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
-export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
-
-# Registry center configuration, determines the type and link of the registry center
-export REGISTRY_TYPE=jdbc
-export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
-export REGISTRY_HIKARI_CONFIG_USERNAME=postgres
-export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres
-
-# Tasks related configurations, need to change the configuration if you use the related tasks.
-export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
-export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
-export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
-export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
-export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
-export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
-
-export MASTER_RESERVED_MEMORY=0.01
-export WORKER_RESERVED_MEMORY=0.01
-
-# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
-#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
-#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
-#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
-#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
-#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS

.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh

@@ -1,58 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# ---------------------------------------------------------
-# INSTALL MACHINE
-# ---------------------------------------------------------
-# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
-# including master, worker, api, alert. If you want to deploy in pseudo-distributed
-# mode, just write a pseudo-distributed hostname
-# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
-ips=${ips:-"localhost"}
-
-# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
-# modify it if you use different ssh port
-sshPort=${sshPort:-"22"}
-
-# A comma separated list of machine hostname or IP would be installed Master server, it
-# must be a subset of configuration `ips`.
-# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
-masters=${masters:-"localhost"}
-
-# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
-# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
-# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
-workers=${workers:-"localhost:default"}
-
-# A comma separated list of machine hostname or IP would be installed Alert server, it
-# must be a subset of configuration `ips`.
-# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
-alertServer=${alertServer:-"localhost"}
-
-# A comma separated list of machine hostname or IP would be installed API server, it
-# must be a subset of configuration `ips`.
-# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
-apiServers=${apiServers:-"localhost"}
-
-# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
-# Do not set this configuration same as the current path (pwd)
-installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
-
-# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
-# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
-# to be created by this user
-deployUser=${deployUser:-"dolphinscheduler"}

.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -euox pipefail
-
-#Start base service containers
-docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d
-
-#Build ds postgresql cluster image
-docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile .
-
-#Start ds postgresql cluster container
-docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d
-
-#Running tests
-/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
-
-#Cleanup
-docker rm -f $(docker ps -aq)

.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml

@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: "3"
-
-services:
-  ds:
-    container_name: ds
-    image: jdk8:ds_postgresql_cluster
-    restart: always
-    ports:
-      - "12345:12345"
-      - "5679:5679"
-      - "1235:1235"
-      - "50053:50053"

.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh

@@ -1,109 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -x
-
-
-API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
-MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
-WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
-ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
-
-#Cluster start health check
-TIMEOUT=180
-START_HEALTHCHECK_EXITCODE=0
-
-for ((i=1; i<=TIMEOUT; i++))
-do
-  MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
-  WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
-  API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
-  ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
-  if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
-    START_HEALTHCHECK_EXITCODE=0
-  else
-    START_HEALTHCHECK_EXITCODE=2
-  fi
-
-  if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
-    echo "cluster start health check success"
-    break
-  fi
-
-  if [[ $i -eq $TIMEOUT ]];then
-      if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
-        echo "master start health check failed"
-      fi
-      if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
-        echo "worker start health check failed"
-      fi
-      if [[ $API_HTTP_STATUS -ne 200 ]]; then
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
-        echo "api start health check failed"
-      fi
-      if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
-        docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
-        echo "alert start health check failed"
-      fi
-      exit $START_HEALTHCHECK_EXITCODE
-    fi
-
-  sleep 1
-done
-
-#Stop Cluster
-docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
-
-#Cluster stop health check
-sleep 5
-MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
-if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-  echo "master stop health check success"
-else
-  echo "master stop health check failed"
-  exit 3
-fi
-
-WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
-if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
-  echo "worker stop health check success"
-else
-  echo "worker stop health check failed"
-  exit 3
-fi
-
-API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
-if [[ $API_HTTP_STATUS -ne 200 ]];then
-  echo "api stop health check success"
-else
-  echo "api stop health check failed"
-  exit 3
-fi
-
-ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
-if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
-  echo "alert stop health check success"
-else
-  echo "alert stop health check failed"
-  exit 3
-fi

.github/workflows/mergeable.yml

@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
----
-#version: 2
-
-on:
-  pull_request:
-
-name: "Mergeable"
-
-jobs:
-  result:
-    name: "Mergeable: milestone-label-check"
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - name: Status
-        run: |
-          echo "Temporary skipping this check"
-
-#mergeable:
-#  # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
-#  # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
-#  # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
-#  # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
-#  - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
-#    name: sync-sql-ddl
-#    validate:
-#      # Sql files must change synchronize
-#      - do: dependent
-#        files:
-#          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
-#          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
-#          - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
-#        message: 'Sql files not change synchronize'
-#    # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
-#    fail:
-#      - do: comment
-#        payload:
-#          body: >
-#            :warning: This PR do not change database DDL synchronize.
-#        leave_old_comment: false
-#      - do: labels
-#        add: 'sql not sync'
-#    # Remove labels 'sql not sync' if pass
-#    pass:
-#      - do: labels
-#        delete: 'sql not sync'
-#
-#  - when: pull_request.*
-#    name: milestone-label-check
-#    validate:
-#      - do: milestone
-#        no_empty:
-#          enabled: false # Cannot be empty when true.
-#          message: 'Milestone is required and cannot be empty.'
-#      - do: label
-#        and:
-#          - must_include:
-#              regex: 'feature|bug|improvement|document|chore|revert'
-#              message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
-#          - must_include:
-#              regex: 'ready-to-merge'
-#              message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'

.github/workflows/unit-test.yml

@@ -76,7 +76,7 @@ jobs:
           restore-keys: ${{ runner.os }}-maven-
 
       - name: Run Unit tests
-        run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false
+        run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false
       - name: Upload coverage report to codecov
         run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
 
@@ -99,11 +99,23 @@ jobs:
           -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
           -Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/*
           -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-          -DskipUT=true
+          -DskipUT=true -DskipIT=true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
+      - name: Collect logs
+        continue-on-error: true
+        run: |
+          mkdir -p ${LOG_DIR}
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        continue-on-error: true
+        with:
+          name: unit-test-logs
+          path: ${LOG_DIR}
   result:
     name: Unit Test
     runs-on: ubuntu-latest

deploy/kubernetes/dolphinscheduler/README.md

@@ -120,12 +120,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
 | conf.common."alert.rpc.port" | int | `50052` | rpc port |
 | conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
-| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` |  |
-| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
-| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
-| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
-| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
-| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required |
 | conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
 | conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
 | conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
@@ -144,6 +138,11 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
 | conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
 | conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
+| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
+| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
+| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
+| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
+| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
 | conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
 | conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
 | conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
@@ -159,7 +158,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.common."yarn.application.status.address" | string | `"http://ds1:%s/ws/v1/cluster/apps/%s"` | if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname |
 | conf.common."yarn.job.history.status.address" | string | `"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"` | job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) |
 | conf.common."yarn.resourcemanager.ha.rm.ids" | string | `"192.168.xx.xx,192.168.xx.xx"` | if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty |
-| datasource.profile | string | `"postgresql"` | The profile of datasource |
 | externalDatabase.database | string | `"dolphinscheduler"` | The database of external database |
 | externalDatabase.driverClassName | string | `"org.postgresql.Driver"` | The driverClassName of external database |
 | externalDatabase.enabled | bool | `false` | If exists external database, and set postgresql.enable value to false. external database will be used, otherwise Dolphinscheduler's internal database will be used. |
@@ -176,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | image.master | string | `"dolphinscheduler-master"` | master image |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent |
 | image.pullSecret | string | `""` | Specify a imagePullSecrets |
-| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler |
+| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler |
 | image.tag | string | `"latest"` | Docker image version for the DolphinScheduler |
 | image.tools | string | `"dolphinscheduler-tools"` | tools image |
 | image.worker | string | `"dolphinscheduler-worker"` | worker image |
@@ -202,9 +200,9 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | master.env.MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER | string | `"true"` | Master kill application when handle failover |
 | master.env.MASTER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Master max heartbeat interval |
 | master.env.MASTER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open master overload protection |
+| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. |
 | master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
-| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. |
+| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. |
-| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. |
 | master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
 | master.env.MASTER_STATE_WHEEL_INTERVAL | string | `"5s"` | master state wheel interval, the unit is second |
 | master.env.MASTER_TASK_COMMIT_INTERVAL | string | `"1s"` | master commit task interval, the unit is second |
@@ -303,11 +301,12 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | worker.env.WORKER_HOST_WEIGHT | string | `"100"` | Worker host weight to dispatch tasks |
 | worker.env.WORKER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Worker heartbeat interval |
 | worker.env.WORKER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open worker overload protection |
+| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. |
 | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. |
-| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. |
+| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. |
-| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. |
 | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. |
 | worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
+| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. |
 | worker.keda.advanced | object | `{}` | Specify HPA related options |
 | worker.keda.cooldownPeriod | int | `30` | How many seconds KEDA will wait before scaling to zero. Note that HPA has a separate cooldown period for scale-downs |
 | worker.keda.enabled | bool | `false` | Enable or disable the Keda component |

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -51,6 +51,7 @@ Create a default common labels.
 {{- define "dolphinscheduler.common.labels" -}}
 app.kubernetes.io/instance: {{ .Release.Name }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/version: {{ .Chart.AppVersion }}
 {{- end -}}
 
 {{/*
@@ -145,10 +146,6 @@ Create a database environment variables.
   {{- else }}
   value: {{ .Values.externalDatabase.type | quote }}
   {{- end }}
-{{- if or .Values.mysql.enabled (eq .Values.externalDatabase.type "mysql") }}
-- name: SPRING_PROFILES_ACTIVE
-  value: mysql
-{{- end }}
 - name: SPRING_DATASOURCE_URL
   {{- if .Values.postgresql.enabled }}
   value: jdbc:postgresql://{{ template "dolphinscheduler.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresqlDatabase }}?{{ .Values.postgresql.params }}

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@@ -1,115 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if and .Values.alert.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "dolphinscheduler.fullname" . }}-alert
-  labels:
-    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
-    {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
-data:
-  application.yaml: |
-    spring:
-      profiles:
-        active: {{ .Values.datasource.profile }}
-      jackson:
-        time-zone: UTC
-        date-format: "yyyy-MM-dd HH:mm:ss"
-      banner:
-        charset: UTF-8
-      datasource:
-        driver-class-name: org.postgresql.Driver
-        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
-        username: root
-        password: root
-        hikari:
-          connection-test-query: select 1
-          pool-name: DolphinScheduler
-    
-    # Mybatis-plus configuration, you don't need to change it
-    mybatis-plus:
-      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
-      type-aliases-package: org.apache.dolphinscheduler.dao.entity
-      configuration:
-        cache-enabled: false
-        call-setters-on-nulls: true
-        map-underscore-to-camel-case: true
-        jdbc-type-for-null: NULL
-      global-config:
-        db-config:
-          id-type: auto
-        banner: false
-    
-    server:
-      port: 50053
-    
-    management:
-      endpoints:
-        web:
-          exposure:
-            include: health,metrics,prometheus
-      endpoint:
-        health:
-          enabled: true
-          show-details: always
-      health:
-        db:
-          enabled: true
-        defaults:
-          enabled: false
-      metrics:
-        tags:
-          application: ${spring.application.name}
-    
-    alert:
-      port: 50052
-      # Mark each alert of alert server if late after x milliseconds as failed.
-      # Define value is (0 = infinite), and alert server would be waiting alert result.
-      wait-timeout: 0
-      max-heartbeat-interval: 60s
-      query_alert_threshold: 100
-    
-    registry:
-      type: zookeeper
-      zookeeper:
-        namespace: dolphinscheduler
-        connect-string: localhost:2181
-        retry-policy:
-          base-sleep-time: 60ms
-          max-sleep: 300ms
-          max-retries: 5
-        session-timeout: 30s
-        connection-timeout: 9s
-        block-until-connected: 600ms
-        digest: ~
-    
-    metrics:
-      enabled: true
-    
-    # Override by profile
-    ---
-    spring:
-      config:
-        activate:
-          on-profile: mysql
-      datasource:
-        driver-class-name: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
-        username: root
-        password: root
-{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml

@@ -1,258 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if and .Values.api.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "dolphinscheduler.fullname" . }}-api
-  labels:
-    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
-    {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
-data:
-  application.yaml: |
-    server:
-      port: 12345
-      servlet:
-        session:
-          timeout: 120m
-        context-path: /dolphinscheduler/
-      compression:
-        enabled: true
-        mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
-      jetty:
-        max-http-form-post-size: 5000000
-        accesslog:
-          enabled: true
-          custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
-    
-    spring:
-      profiles:
-        active: {{ .Values.datasource.profile }}
-      banner:
-        charset: UTF-8
-      jackson:
-        time-zone: UTC
-        date-format: "yyyy-MM-dd HH:mm:ss"
-      servlet:
-        multipart:
-          max-file-size: 1024MB
-          max-request-size: 1024MB
-      messages:
-        basename: i18n/messages
-      datasource:
-        driver-class-name: org.postgresql.Driver
-        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
-        username: root
-        password: root
-        hikari:
-          connection-test-query: select 1
-          pool-name: DolphinScheduler
-      quartz:
-        auto-startup: false
-        job-store-type: jdbc
-        jdbc:
-          initialize-schema: never
-        properties:
-          org.quartz.jobStore.isClustered: true
-          org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
-          org.quartz.scheduler.instanceId: AUTO
-          org.quartz.jobStore.tablePrefix: QRTZ_
-          org.quartz.jobStore.acquireTriggersWithinLock: true
-          org.quartz.scheduler.instanceName: DolphinScheduler
-          org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
-          org.quartz.jobStore.useProperties: false
-          org.quartz.jobStore.misfireThreshold: 60000
-          org.quartz.scheduler.makeSchedulerThreadDaemon: true
-          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
-          org.quartz.jobStore.clusterCheckinInterval: 5000
-          org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
-      mvc:
-        pathmatch:
-          matching-strategy: ANT_PATH_MATCHER
-        static-path-pattern: /static/**
-    springdoc:
-      swagger-ui:
-        path: /swagger-ui.html
-      packages-to-scan: org.apache.dolphinscheduler.api
-    
-    # Mybatis-plus configuration, you don't need to change it
-    mybatis-plus:
-      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
-      type-aliases-package: org.apache.dolphinscheduler.dao.entity
-      configuration:
-        cache-enabled: false
-        call-setters-on-nulls: true
-        map-underscore-to-camel-case: true
-        jdbc-type-for-null: NULL
-      global-config:
-        db-config:
-          id-type: auto
-        banner: false
-    
-    management:
-      endpoints:
-        web:
-          exposure:
-            include: health,metrics,prometheus
-      endpoint:
-        health:
-          enabled: true
-          show-details: always
-      health:
-        db:
-          enabled: true
-        defaults:
-          enabled: false
-      metrics:
-        tags:
-          application: ${spring.application.name}
-    
-    registry:
-      type: zookeeper
-      zookeeper:
-        namespace: dolphinscheduler
-        connect-string: localhost:2181
-        retry-policy:
-          base-sleep-time: 60ms
-          max-sleep: 300ms
-          max-retries: 5
-        session-timeout: 60s
-        connection-timeout: 15s
-        block-until-connected: 15s
-        digest: ~
-    
-    api:
-      audit-enable: false
-      # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
-      # global max request number per second
-      # default tenant-level max request number
-      traffic-control:
-        global-switch: false
-        max-global-qps-rate: 300
-        tenant-switch: false
-        default-tenant-qps-rate: 10
-          #customize-tenant-qps-rate:
-        # eg.
-        #tenant1: 11
-        #tenant2: 20
-      python-gateway:
-        # Weather enable python gateway server or not. The default value is false.
-        enabled: false
-        # Authentication token for connection from python api to python gateway server. Should be changed the default value
-        # when you deploy in public network.
-        auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
-        # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
-        # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
-        gateway-server-address: 0.0.0.0
-        # The port of Python gateway server start. Define which port you could connect to Python gateway server from
-        # Python API side.
-        gateway-server-port: 25333
-        # The address of Python callback client.
-        python-address: 127.0.0.1
-        # The port of Python callback client.
-        python-port: 25334
-        # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
-        # and socket server would never close even though no requests accept
-        connect-timeout: 0
-        # Close each active connection of socket server if python program not active after x milliseconds. Define value is
-        # (0 = infinite), and socket server would never close even though no requests accept
-        read-timeout: 0
-    
-    metrics:
-      enabled: true
-    
-    security:
-      authentication:
-        # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
-        type: PASSWORD
-        # IF you set type `LDAP`, below config will be effective
-        ldap:
-          # ldap server config
-          urls: ldap://ldap.forumsys.com:389/
-          base-dn: dc=example,dc=com
-          username: cn=read-only-admin,dc=example,dc=com
-          password: password
-          user:
-            # admin userId when you use LDAP login
-            admin: read-only-admin
-            identity-attribute: uid
-            email-attribute: mail
-            # action when ldap user is not exist (supported types: CREATE,DENY)
-            not-exist-action: CREATE
-          ssl:
-            enable: false
-            # jks file absolute path && password
-            trust-store: "/ldapkeystore.jks"
-            trust-store-password: "password"
-        casdoor:
-          user:
-            admin: ""
-        oauth2:
-          enable: false
-          provider:
-            github:
-              authorizationUri: ""
-              redirectUri: ""
-              clientId: ""
-              clientSecret: ""
-              tokenUri: ""
-              userInfoUri: ""
-              callbackUrl: ""
-              iconUri: ""
-              provider: github
-            google:
-              authorizationUri: ""
-              redirectUri: ""
-              clientId: ""
-              clientSecret: ""
-              tokenUri: ""
-              userInfoUri: ""
-              callbackUrl: ""
-              iconUri: ""
-              provider: google
-    casdoor:
-      # Your Casdoor server url
-      endpoint: ""
-      client-id: ""
-      client-secret: ""
-      # The certificate may be multi-line, you can use `|-` for ease
-      certificate: ""
-      # Your organization name added in Casdoor
-      organization-name: ""
-      # Your application name added in Casdoor
-      application-name: ""
-      # Doplhinscheduler login url
-      redirect-url: ""
-    
-    # Override by profile
-    ---
-    spring:
-      config:
-        activate:
-          on-profile: mysql
-      datasource:
-        driver-class-name: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
-        username: root
-        password: root
-      quartz:
-        properties:
-          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-{{- end }}
-
-
-

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml

@@ -1,173 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if and .Values.master.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "dolphinscheduler.fullname" . }}-master
-  labels:
-    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
-    {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
-data:
-  application.yaml: |
-    spring:
-      profiles:
-        active: {{ .Values.datasource.profile }}
-      banner:
-        charset: UTF-8
-      jackson:
-        time-zone: UTC
-        date-format: "yyyy-MM-dd HH:mm:ss"
-      datasource:
-        driver-class-name: org.postgresql.Driver
-        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
-        username: root
-        password: root
-        hikari:
-          connection-test-query: select 1
-          pool-name: DolphinScheduler
-      quartz:
-        job-store-type: jdbc
-        jdbc:
-          initialize-schema: never
-        properties:
-          org.quartz.threadPool.threadPriority: 5
-          org.quartz.jobStore.isClustered: true
-          org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
-          org.quartz.scheduler.instanceId: AUTO
-          org.quartz.jobStore.tablePrefix: QRTZ_
-          org.quartz.jobStore.acquireTriggersWithinLock: true
-          org.quartz.scheduler.instanceName: DolphinScheduler
-          org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
-          org.quartz.jobStore.useProperties: false
-          org.quartz.threadPool.makeThreadsDaemons: true
-          org.quartz.threadPool.threadCount: 25
-          org.quartz.jobStore.misfireThreshold: 60000
-          org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
-          org.quartz.scheduler.makeSchedulerThreadDaemon: true
-          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
-          org.quartz.jobStore.clusterCheckinInterval: 5000
-    
-    # Mybatis-plus configuration, you don't need to change it
-    mybatis-plus:
-      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
-      type-aliases-package: org.apache.dolphinscheduler.dao.entity
-      configuration:
-        cache-enabled: false
-        call-setters-on-nulls: true
-        map-underscore-to-camel-case: true
-        jdbc-type-for-null: NULL
-      global-config:
-        db-config:
-          id-type: auto
-        banner: false
-    
-    
-    registry:
-      type: zookeeper
-      zookeeper:
-        namespace: dolphinscheduler
-        connect-string: localhost:2181
-        retry-policy:
-          base-sleep-time: 60ms
-          max-sleep: 300ms
-          max-retries: 5
-        session-timeout: 30s
-        connection-timeout: 9s
-        block-until-connected: 600ms
-        digest: ~
-    
-    master:
-      listen-port: 5678
-      # master fetch command num
-      fetch-command-num: 10
-      # master prepare execute thread number to limit handle commands in parallel
-      pre-exec-threads: 10
-      # master execute thread number to limit process instances in parallel
-      exec-threads: 100
-      # master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s.
-      dispatch-task-number: 3
-      # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
-      host-selector: lower_weight
-      # master heartbeat interval
-      max-heartbeat-interval: 10s
-      # master commit task retry times
-      task-commit-retry-times: 5
-      # master commit task interval
-      task-commit-interval: 1s
-      state-wheel-interval: 5s
-      server-load-protection:
-        # If set true, will open master overload protection
-        enabled: true
-        # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
-        max-system-cpu-usage-percentage-thresholds: 0.7
-        # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
-        max-jvm-cpu-usage-percentage-thresholds: 0.7
-        # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
-        max-system-memory-usage-percentage-thresholds: 0.7
-        # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
-        max-disk-usage-percentage-thresholds: 0.7
-      # failover interval, the unit is minute
-      failover-interval: 10m
-      # kill yarn / k8s application when failover taskInstance, default true
-      kill-application-when-task-failover: true
-      registry-disconnect-strategy:
-        # The disconnect strategy: stop, waiting
-        strategy: waiting
-        # The max waiting time to reconnect to registry if you set the strategy to waiting
-        max-waiting-time: 100s
-      worker-group-refresh-interval: 10s
-    
-    server:
-      port: 5679
-    
-    management:
-      endpoints:
-        web:
-          exposure:
-            include: health,metrics,prometheus
-      endpoint:
-        health:
-          enabled: true
-          show-details: always
-      health:
-        db:
-          enabled: true
-        defaults:
-          enabled: false
-      metrics:
-        tags:
-          application: ${spring.application.name}
-    
-    metrics:
-      enabled: true
-    
-    # Override by profile
-    ---
-    spring:
-      config:
-        activate:
-          on-profile: mysql
-      datasource:
-        driver-class-name: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
-        username: root
-        password: root
-      quartz:
-        properties:
-          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@@ -1,108 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if and .Values.worker.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "dolphinscheduler.fullname" . }}-worker
-  labels:
-    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
-    {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
-data:
-  application.yaml: |
-    spring:
-      banner:
-        charset: UTF-8
-      jackson:
-        time-zone: UTC
-        date-format: "yyyy-MM-dd HH:mm:ss"
-      autoconfigure:
-        exclude:
-          - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
-    
-    registry:
-      type: zookeeper
-      zookeeper:
-        namespace: dolphinscheduler
-        connect-string: localhost:2181
-        retry-policy:
-          base-sleep-time: 60ms
-          max-sleep: 300ms
-          max-retries: 5
-        session-timeout: 30s
-        connection-timeout: 9s
-        block-until-connected: 600ms
-        digest: ~
-    
-    worker:
-      # worker listener port
-      listen-port: 1234
-      # worker execute thread number to limit task instances in parallel
-      exec-threads: 100
-      # worker heartbeat interval
-      max-heartbeat-interval: 10s
-      # worker host weight to dispatch tasks, default value 100
-      host-weight: 100
-      server-load-protection:
-        # If set true, will open worker overload protection
-        enabled: true
-        # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
-        max-system-cpu-usage-percentage-thresholds: 0.7
-        # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
-        max-jvm-cpu-usage-percentage-thresholds: 0.7
-        # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
-        max-system-memory-usage-percentage-thresholds: 0.7
-        # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
-        max-disk-usage-percentage-thresholds: 0.7
-      registry-disconnect-strategy:
-        # The disconnect strategy: stop, waiting
-        strategy: waiting
-        # The max waiting time to reconnect to registry if you set the strategy to waiting
-        max-waiting-time: 100s
-      task-execute-threads-full-policy: REJECT
-      tenant-config:
-        # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
-        auto-create-tenant-enabled: true
-        # Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants.
-        distributed-tenant-enabled: false
-        # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
-        default-tenant-enabled: false
-    
-    server:
-      port: 1235
-    
-    management:
-      endpoints:
-        web:
-          exposure:
-            include: health,metrics,prometheus
-      endpoint:
-        health:
-          enabled: true
-          show-details: always
-      health:
-        db:
-          enabled: true
-        defaults:
-          enabled: false
-      metrics:
-        tags:
-          application: ${spring.application.name}
-    
-    metrics:
-      enabled: true
-{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@@ -32,12 +32,12 @@ data:
       {{- end }}
     {{- end }}
   {{- end }}
-  common.properties: |-
+  common_properties: |-
   {{- if index .Values.conf "common" }}
   {{- range $key, $value := index .Values.conf "common" }}
     {{- if and $.Values.minio.enabled }}
     {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
-    {{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
+    {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
     {{- end }}
     {{ $key }}={{ $value }}
   {{- end }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -114,10 +114,7 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-alert
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common.properties
+              subPath: common_properties
-            - name: alert-config-volume
-              mountPath: /opt/dolphinscheduler/conf/application.yaml
-              subPath: application.yaml
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-alert
           {{- if .Values.alert.persistentVolumeClaim.enabled }}
@@ -129,7 +126,4 @@ spec:
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
-        - name: alert-config-volume
-          configMap:
-            name: {{ include "dolphinscheduler.fullname" . }}-alert
 {{- end }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -115,15 +115,12 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-api
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common.properties
+              subPath: common_properties
             {{- if .Values.api.taskTypeFilter.enabled }}
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
               subPath: task-type-config.yaml
             {{- end }}
-            - name: api-config-volume
-              mountPath: /opt/dolphinscheduler/conf/application.yaml
-              subPath: application.yaml
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
@@ -139,9 +136,6 @@ spec:
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
-        - name: api-config-volume
-          configMap:
-            name: {{ include "dolphinscheduler.fullname" . }}-api
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -109,13 +109,10 @@ spec:
           volumeMounts:
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-master
-            - name: master-config-volume
-              mountPath: /opt/dolphinscheduler/conf/application.yaml
-              subPath: application.yaml
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common.properties
+              subPath: common_properties
             {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-master
@@ -125,9 +122,6 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        - name: master-config-volume
-          configMap:
-            name: {{ include "dolphinscheduler.fullname" . }}-master
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         - name: config-volume
           configMap:

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -111,12 +111,9 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-worker-data
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
-            - name: worker-config-volume
-              mountPath: /opt/dolphinscheduler/conf/application.yaml
-              subPath: application.yaml
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common.properties
+              subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
@@ -142,9 +139,6 @@ spec:
         - name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
           emptyDir: {}
         {{- end }}
-        - name: worker-config-volume
-          configMap:
-            name: {{ include "dolphinscheduler.fullname" . }}-worker
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -31,7 +31,7 @@ initImage:
 
 image:
   # -- Docker image repository for the DolphinScheduler
-  registry: apache
+  registry: apache/dolphinscheduler
   # -- Docker image version for the DolphinScheduler
   tag: latest
   # -- Image pull policy. Options: Always, Never, IfNotPresent
@@ -49,10 +49,6 @@ image:
   # -- tools image
   tools: dolphinscheduler-tools
 
-datasource:
-  # -- The profile of datasource
-  profile: postgresql
-
 postgresql:
   # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL
   enabled: true
@@ -250,25 +246,20 @@ conf:
     # -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
     resource.storage.upload.base.path: /dolphinscheduler
 
-    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    resource.aws.access.key.id: minioadmin
-    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
-    aws.credentials.provider.type: AWSStaticCredentialsProvider
 
-    # -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
+    # -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    aws.s3.access.key.id: minioadmin
+    resource.aws.secret.access.key: minioadmin
 
-    # -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
+    # -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    aws.s3.access.key.secret: minioadmin
+    resource.aws.region: ca-central-1
-
-    # -- The AWS Region to use. if resource.storage.type=S3, This configuration is required
-    aws.s3.region: ca-central-1
 
     # -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-    aws.s3.bucket.name: dolphinscheduler
+    resource.aws.s3.bucket.name: dolphinscheduler
 
     # -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-    aws.s3.endpoint: http://minio:9000
+    resource.aws.s3.endpoint: http://minio:9000
 
     # -- alibaba cloud access key id, required if you set resource.storage.type=OSS
     resource.alibaba.cloud.access.key.id: <your-access-key-id>
@@ -517,10 +508,10 @@ master:
     MASTER_STATE_WHEEL_INTERVAL: "5s"
     # -- If set true, will open master overload protection
     MASTER_SERVER_LOAD_PROTECTION_ENABLED: false
-    # -- Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
+    # -- Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow.
-    MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
+    MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
-    # -- Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
+    # -- Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow.
-    MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
+    MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
     # -- Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
     MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
     # -- Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
@@ -638,10 +629,10 @@ worker:
   env:
     # -- If set true, will open worker overload protection
     WORKER_SERVER_LOAD_PROTECTION_ENABLED: false
-    # -- Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
+    # -- Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks.
-    WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
+    WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
-    # -- Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
+    # -- Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks.
-    WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
+    WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
     # -- Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks.
     WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
     # -- Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
@@ -654,6 +645,8 @@ worker:
     WORKER_HOST_WEIGHT: "100"
     # -- tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
     WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: true
+    # -- Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
+    WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT: false
     # -- If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`;
     DEFAULT_TENANT_ENABLED: false
 

docs/docs/en/architecture/configuration.md

@@ -110,8 +110,7 @@ The directory structure of DolphinScheduler is as follows:
 
 dolphinscheduler-daemon.sh is responsible for DolphinScheduler startup and shutdown.
 Essentially, start-all.sh or stop-all.sh startup and shutdown the cluster via dolphinscheduler-daemon.sh.
-Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical
+Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical situation of resources.
-situation of resources.
 
 Default simplified parameters are:
 
@@ -129,47 +128,44 @@ export DOLPHINSCHEDULER_OPTS="
 "
 ```
 
-> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to
+> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to communicate).
-> communicate).
+> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6.
-> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will
-> use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6.
 
 ### Database connection related configuration
 
 DolphinScheduler uses Spring Hikari to manage database connections, configuration file location:
 
-|    Service    |          Configuration file           |
+|Service| Configuration file  |
-|---------------|---------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/application.yaml` |
+|Master Server | `master-server/conf/application.yaml`|
-| Api Server    | `api-server/conf/application.yaml`    |
+|Api Server| `api-server/conf/application.yaml`|
-| Worker Server | `worker-server/conf/application.yaml` |
+|Worker Server| `worker-server/conf/application.yaml`|
-| Alert Server  | `alert-server/conf/application.yaml`  |
+|Alert Server| `alert-server/conf/application.yaml`|
 
 The default configuration is as follows:
 
-|                      Parameters                      |                   Default value                   |                  Description                  |
+|Parameters | Default value| Description|
-|------------------------------------------------------|---------------------------------------------------|-----------------------------------------------|
+|--|--|--|
-| spring.datasource.driver-class-name                  | org.postgresql.Driver                             | datasource driver                             |
+|spring.datasource.driver-class-name| org.postgresql.Driver |datasource driver|
-| spring.datasource.url                                | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | datasource connection url                     |
+|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |datasource connection url|
-| spring.datasource.username                           | root                                              | datasource username                           |
+|spring.datasource.username|root|datasource username|
-| spring.datasource.password                           | root                                              | datasource password                           |
+|spring.datasource.password|root|datasource password|
-| spring.datasource.hikari.connection-test-query       | select 1                                          | validate connection by running the SQL        |
+|spring.datasource.hikari.connection-test-query|select 1|validate connection by running the SQL|
-| spring.datasource.hikari.minimum-idle                | 5                                                 | minimum connection pool size number           |
+|spring.datasource.hikari.minimum-idle| 5| minimum connection pool size number|
-| spring.datasource.hikari.auto-commit                 | true                                              | whether auto commit                           |
+|spring.datasource.hikari.auto-commit|true|whether auto commit|
-| spring.datasource.hikari.pool-name                   | DolphinScheduler                                  | name of the connection pool                   |
+|spring.datasource.hikari.pool-name|DolphinScheduler|name of the connection pool|
-| spring.datasource.hikari.maximum-pool-size           | 50                                                | maximum connection pool size number           |
+|spring.datasource.hikari.maximum-pool-size|50| maximum connection pool size number|
-| spring.datasource.hikari.connection-timeout          | 30000                                             | connection timeout                            |
+|spring.datasource.hikari.connection-timeout|30000|connection timeout|
-| spring.datasource.hikari.idle-timeout                | 600000                                            | Maximum idle connection survival time         |
+|spring.datasource.hikari.idle-timeout|600000|Maximum idle connection survival time|
-| spring.datasource.hikari.leak-detection-threshold    | 0                                                 | Connection leak detection threshold           |
+|spring.datasource.hikari.leak-detection-threshold|0|Connection leak detection threshold|
-| spring.datasource.hikari.initialization-fail-timeout | 1                                                 | Connection pool initialization failed timeout |
+|spring.datasource.hikari.initialization-fail-timeout|1|Connection pool initialization failed timeout|
 
 Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`.
 
-### Registry Related configuration
+### Zookeeper related configuration
 
-DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions.
+DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions. Configuration file location:
-Configuration file location:
+|Service| Configuration file  |
-|Service| Configuration file |
 |--|--|
 |Master Server | `master-server/conf/application.yaml`|
 |Api Server| `api-server/conf/application.yaml`|
@@ -177,35 +173,30 @@ Configuration file location:
 
 The default configuration is as follows:
 
-|                   Parameters                    |     Default value     |                                                                                       Description                                                                                       |
+|Parameters | Default value| Description|
-|-------------------------------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|--|--|--|
-| registry.zookeeper.namespace                    | dolphinscheduler      | namespace of zookeeper                                                                                                                                                                  |
+|registry.zookeeper.namespace|dolphinscheduler|namespace of zookeeper|
-| registry.zookeeper.connect-string               | localhost:2181        | the connection string of zookeeper                                                                                                                                                      |
+|registry.zookeeper.connect-string|localhost:2181| the connection string of zookeeper|
-| registry.zookeeper.retry-policy.base-sleep-time | 60ms                  | time to wait between subsequent retries                                                                                                                                                 |
+|registry.zookeeper.retry-policy.base-sleep-time|60ms|time to wait between subsequent retries|
-| registry.zookeeper.retry-policy.max-sleep       | 300ms                 | maximum time to wait between subsequent retries                                                                                                                                         |
+|registry.zookeeper.retry-policy.max-sleep|300ms|maximum time to wait between subsequent retries|
-| registry.zookeeper.retry-policy.max-retries     | 5                     | maximum retry times                                                                                                                                                                     |
+|registry.zookeeper.retry-policy.max-retries|5|maximum retry times|
-| registry.zookeeper.session-timeout              | 30s                   | session timeout                                                                                                                                                                         |
+|registry.zookeeper.session-timeout|30s|session timeout|
-| registry.zookeeper.connection-timeout           | 30s                   | connection timeout                                                                                                                                                                      |
+|registry.zookeeper.connection-timeout|30s|connection timeout|
-| registry.zookeeper.block-until-connected        | 600ms                 | waiting time to block until the connection succeeds                                                                                                                                     |
+|registry.zookeeper.block-until-connected|600ms|waiting time to block until the connection succeeds|
-| registry.zookeeper.digest                       | {username}:{password} | digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) |
+|registry.zookeeper.digest|{username}:{password}|digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) |
 
 Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
 
-For ETCD Registry, please see more details
-on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
-For JDBC Registry, please see more details
-on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
-
 ### common.properties [hadoop、s3、yarn config properties]
 
 Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
 
-|    Service    |                          Configuration file                           |
+|Service| Configuration file  |
-|---------------|-----------------------------------------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/common.properties`                                |
+|Master Server | `master-server/conf/common.properties`|
-| Api Server    | `api-server/conf/common.properties`, `api-server/conf/aws.yaml`       |
+|Api Server| `api-server/conf/common.properties`|
-| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` |
+|Worker Server| `worker-server/conf/common.properties`|
-| Alert Server  | `alert-server/conf/common.properties`                                 |
+|Alert Server| `alert-server/conf/common.properties`|
 
 The default configuration is as follows:
 
@@ -214,6 +205,10 @@ The default configuration is as follows:
 | data.basedir.path                             | /tmp/dolphinscheduler                            | local directory used to store temp files                                                                                                                                                                                                                                                                                                                                                                                             |
 | resource.storage.type                         | NONE                                             | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE                                                                                                                                                                                                                                                                                                                                                                                |
 | resource.upload.path                          | /dolphinscheduler                                | storage path of resource files                                                                                                                                                                                                                                                                                                                                                                                                       |
+| aws.access.key.id                             | minioadmin                                       | access key id of S3                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| aws.secret.access.key                         | minioadmin                                       | secret access key of S3                                                                                                                                                                                                                                                                                                                                                                                                              |
+| aws.region                                    | us-east-1                                        | region of S3                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| aws.s3.endpoint                               | http://minio:9000                                | endpoint of S3                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | hdfs.root.user                                | hdfs                                             | configure users with corresponding permissions if storage type is HDFS                                                                                                                                                                                                                                                                                                                                                               |
 | fs.defaultFS                                  | hdfs://mycluster:8020                            | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory                                                                                                                                                                                                           |
 | hadoop.security.authentication.startup.state  | false                                            | whether hadoop grant kerberos permission                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -242,43 +237,43 @@ The default configuration is as follows:
 
 Location: `api-server/conf/application.yaml`
 
-|                      Parameters                       |            Default value             |                                          Description                                           |
+|Parameters | Default value| Description|
-|-------------------------------------------------------|--------------------------------------|------------------------------------------------------------------------------------------------|
+|--|--|--|
-| server.port                                           | 12345                                | api service communication port                                                                 |
+|server.port|12345|api service communication port|
-| server.servlet.session.timeout                        | 120m                                 | session timeout                                                                                |
+|server.servlet.session.timeout|120m|session timeout|
-| server.servlet.context-path                           | /dolphinscheduler/                   | request path                                                                                   |
+|server.servlet.context-path|/dolphinscheduler/ |request path|
-| spring.servlet.multipart.max-file-size                | 1024MB                               | maximum file size                                                                              |
+|spring.servlet.multipart.max-file-size|1024MB|maximum file size|
-| spring.servlet.multipart.max-request-size             | 1024MB                               | maximum request size                                                                           |
+|spring.servlet.multipart.max-request-size|1024MB|maximum request size|
-| server.jetty.max-http-post-size                       | 5000000                              | jetty maximum post size                                                                        |
+|server.jetty.max-http-post-size|5000000|jetty maximum post size|
-| spring.banner.charset                                 | UTF-8                                | message encoding                                                                               |
+|spring.banner.charset|UTF-8|message encoding|
-| spring.jackson.time-zone                              | UTC                                  | time zone                                                                                      |
+|spring.jackson.time-zone|UTC|time zone|
-| spring.jackson.date-format                            | "yyyy-MM-dd HH:mm:ss"                | time format                                                                                    |
+|spring.jackson.date-format|"yyyy-MM-dd HH:mm:ss"|time format|
-| spring.messages.basename                              | i18n/messages                        | i18n config                                                                                    |
+|spring.messages.basename|i18n/messages|i18n config|
-| security.authentication.type                          | PASSWORD                             | authentication type                                                                            |
+|security.authentication.type|PASSWORD|authentication type|
-| security.authentication.ldap.user.admin               | read-only-admin                      | admin user account when you log-in with LDAP                                                   |
+|security.authentication.ldap.user.admin|read-only-admin|admin user account when you log-in with LDAP|
-| security.authentication.ldap.urls                     | ldap://ldap.forumsys.com:389/        | LDAP urls                                                                                      |
+|security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls|
-| security.authentication.ldap.base.dn                  | dc=example,dc=com                    | LDAP base dn                                                                                   |
+|security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn|
-| security.authentication.ldap.username                 | cn=read-only-admin,dc=example,dc=com | LDAP username                                                                                  |
+|security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username|
-| security.authentication.ldap.password                 | password                             | LDAP password                                                                                  |
+|security.authentication.ldap.password|password|LDAP password|
-| security.authentication.ldap.user.identity-attribute  | uid                                  | LDAP user identity attribute                                                                   |
+|security.authentication.ldap.user.identity-attribute|uid|LDAP user identity attribute|
-| security.authentication.ldap.user.email-attribute     | mail                                 | LDAP user email attribute                                                                      |
+|security.authentication.ldap.user.email-attribute|mail|LDAP user email attribute|
-| security.authentication.ldap.user.not-exist-action    | CREATE                               | action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY) |
+|security.authentication.ldap.user.not-exist-action|CREATE|action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY)|
-| security.authentication.ldap.ssl.enable               | false                                | LDAP ssl switch                                                                                |
+|security.authentication.ldap.ssl.enable|false|LDAP ssl switch|
-| security.authentication.ldap.ssl.trust-store          | ldapkeystore.jks                     | LDAP jks file absolute path                                                                    |
+|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks file absolute path|
-| security.authentication.ldap.ssl.trust-store-password | password                             | LDAP jks password                                                                              |
+|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks password|
-| security.authentication.casdoor.user.admin            |                                      | admin user account when you log-in with Casdoor                                                |
+|security.authentication.casdoor.user.admin||admin user account when you log-in with Casdoor|
-| casdoor.endpoint                                      |                                      | Casdoor server url                                                                             |
+|casdoor.endpoint||Casdoor server url|
-| casdoor.client-id                                     |                                      | id in Casdoor                                                                                  |
+|casdoor.client-id||id in Casdoor|
-| casdoor.client-secret                                 |                                      | secret in Casdoor                                                                              |
+|casdoor.client-secret||secret in Casdoor|
-| casdoor.certificate                                   |                                      | certificate in Casdoor                                                                         |
+|casdoor.certificate||certificate in Casdoor|
-| casdoor.organization-name                             |                                      | organization name in Casdoor                                                                   |
+|casdoor.organization-name||organization name in Casdoor|
-| casdoor.application-name                              |                                      | application name in Casdoor                                                                    |
+|casdoor.application-name||application name in Casdoor|
-| casdoor.redirect-url                                  |                                      | doplhinscheduler login url                                                                     |
+|casdoor.redirect-url||doplhinscheduler login url|
-| api.traffic.control.global.switch                     | false                                | traffic control global switch                                                                  |
+|api.traffic.control.global.switch|false|traffic control global switch|
-| api.traffic.control.max-global-qps-rate               | 300                                  | global max request number per second                                                           |
+|api.traffic.control.max-global-qps-rate|300|global max request number per second|
-| api.traffic.control.tenant-switch                     | false                                | traffic control tenant switch                                                                  |
+|api.traffic.control.tenant-switch|false|traffic control tenant switch|
-| api.traffic.control.default-tenant-qps-rate           | 10                                   | default tenant max request number per second                                                   |
+|api.traffic.control.default-tenant-qps-rate|10|default tenant max request number per second|
-| api.traffic.control.customize-tenant-qps-rate         |                                      | customize tenant max request number per second                                                 |
+|api.traffic.control.customize-tenant-qps-rate||customize tenant max request number per second|
 
 ### Master Server related configuration
 
@@ -287,6 +282,7 @@ Location: `master-server/conf/application.yaml`
 |                                 Parameters                                  | Default value |                                                                                                                                                         Description                                                                                                                                                          |
 |-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | master.listen-port                                                          | 5678          | master listen port                                                                                                                                                                                                                                                                                                           |
+| master.fetch-command-num                                                    | 10            | the number of commands fetched by master                                                                                                                                                                                                                                                                                     |
 | master.pre-exec-threads                                                     | 10            | master prepare execute thread number to limit handle commands in parallel                                                                                                                                                                                                                                                    |
 | master.exec-threads                                                         | 100           | master execute thread number to limit process instances in parallel                                                                                                                                                                                                                                                          |
 | master.dispatch-task-number                                                 | 3             | master dispatch task number per batch                                                                                                                                                                                                                                                                                        |
@@ -296,39 +292,37 @@ Location: `master-server/conf/application.yaml`
 | master.task-commit-interval                                                 | 1000          | master commit task interval, the unit is millisecond                                                                                                                                                                                                                                                                         |
 | master.state-wheel-interval                                                 | 5             | time to check status                                                                                                                                                                                                                                                                                                         |
 | master.server-load-protection.enabled                                       | true          | If set true, will open master overload protection                                                                                                                                                                                                                                                                            |
-| master.server-load-protection.max-system-cpu-usage-percentage-thresholds    | 0.7           | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                              |
+| master.server-load-protection.max-cpu-usage-percentage-thresholds           | 0.7           | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                                            |
-| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds       | 0.7           | Master max JVM cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                                    |
+| master.server-load-protection.max-jvm-memory-usage-percentage-thresholds    | 0.7           | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                             |
-| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7           | Master max system memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                       |
+| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7           | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                       |
 | master.server-load-protection.max-disk-usage-percentage-thresholds          | 0.7           | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                                         |
 | master.failover-interval                                                    | 10            | failover interval, the unit is minute                                                                                                                                                                                                                                                                                        |
 | master.kill-application-when-task-failover                                  | true          | whether to kill yarn/k8s application when failover taskInstance                                                                                                                                                                                                                                                              |
 | master.registry-disconnect-strategy.strategy                                | stop          | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting                                                                                                                                                                                                                    |
 | master.registry-disconnect-strategy.max-waiting-time                        | 100s          | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely |
 | master.worker-group-refresh-interval                                        | 10s           | The interval to refresh worker group from db to memory                                                                                                                                                                                                                                                                       |
-| master.command-fetch-strategy.type                                          | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED`                                                                                                                                                                                                                                                                     |
-| master.command-fetch-strategy.config.id-step                                | 1             | The id auto incremental step of t_ds_command in db                                                                                                                                                                                                                                                                           |
-| master.command-fetch-strategy.config.fetch-size                             | 10            | The number of commands fetched by master                                                                                                                                                                                                                                                                                     |
 
 ### Worker Server related configuration
 
 Location: `worker-server/conf/application.yaml`
 
-|                                 Parameters                                  | Default value |                                                                                                                                                    Description                                                                                                                                                    |
+|                                   Parameters                                   | Default value |                                                                                                                                                    Description                                                                                                                                                    |
-|-----------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|--------------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| worker.listen-port                                                          | 1234          | worker-service listen port                                                                                                                                                                                                                                                                                        |
+| worker.listen-port                                                             | 1234          | worker-service listen port                                                                                                                                                                                                                                                                                        |
-| worker.exec-threads                                                         | 100           | worker-service execute thread number, used to limit the number of task instances in parallel                                                                                                                                                                                                                      |
+| worker.exec-threads                                                            | 100           | worker-service execute thread number, used to limit the number of task instances in parallel                                                                                                                                                                                                                      |
-| worker.max-heartbeat-interval                                               | 10s           | worker-service max heartbeat interval                                                                                                                                                                                                                                                                             |
+| worker.max-heartbeat-interval                                                  | 10s           | worker-service max heartbeat interval                                                                                                                                                                                                                                                                             |
-| worker.host-weight                                                          | 100           | worker host weight to dispatch tasks                                                                                                                                                                                                                                                                              |
+| worker.host-weight                                                             | 100           | worker host weight to dispatch tasks                                                                                                                                                                                                                                                                              |
-| worker.server-load-protection.enabled                                       | true          | If set true will open worker overload protection                                                                                                                                                                                                                                                                  |
+| worker.server-load-protection.enabled                                          | true          | If set true will open worker overload protection                                                                                                                                                                                                                                                                  |
-| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds    | 0.7           | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                   |
+| worker.max-cpu-usage-percentage-thresholds.max-cpu-usage-percentage-thresholds | 0.7           | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                                 |
-| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds       | 0.7           | Worker max JVM cpu usage, when the worker's jvm cpu usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                         |
+| worker.server-load-protection.max-jvm-memory-usage-percentage-thresholds       | 0.7           | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                  |
-| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7           | Worker max system memory usage , when the worker's system memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                            |
+| worker.server-load-protection.max-system-memory-usage-percentage-thresholds    | 0.7           | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.                                                                                                                                                                            |
-| worker.server-load-protection.max-disk-usage-percentage-thresholds          | 0.7           | Worker max disk usage , when the worker's disk usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                              |
+| worker.server-load-protection.max-disk-usage-percentage-thresholds             | 0.7           | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.                                                                                                                                                                                              |
-| worker.registry-disconnect-strategy.strategy                                | stop          | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting                                                                                                                                                                                                         |
+| worker.registry-disconnect-strategy.strategy                                   | stop          | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting                                                                                                                                                                                                         |
-| worker.registry-disconnect-strategy.max-waiting-time                        | 100s          | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
+| worker.registry-disconnect-strategy.max-waiting-time                           | 100s          | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
-| worker.task-execute-threads-full-policy                                     | REJECT        | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution                                                        |
+| worker.task-execute-threads-full-policy                                        | REJECT        | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution                                                        |
-| worker.tenant-config.auto-create-tenant-enabled                             | true          | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.                                                                                           |
+| worker.tenant-config.auto-create-tenant-enabled                                | true          | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.                                                                                           |
-| worker.tenant-config.default-tenant-enabled                                 | false         | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.                                                                                                                                                                                                           |
+| worker.tenant-config.distributed-tenant-enabled                                | false         | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants                                                                                                                                                                                                   |
+| worker.tenant-config.default-tenant-enabled                                    | false         | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.                                                                                                                                                                                                           |
 
 ### Alert Server related configuration
 
@@ -343,10 +337,10 @@ Location: `alert-server/conf/application.yaml`
 
 This part describes quartz configs and configure them based on your practical situation and resources.
 
-|    Service    |          Configuration file           |
+|Service| Configuration file  |
-|---------------|---------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/application.yaml` |
+|Master Server | `master-server/conf/application.yaml`|
-| Api Server    | `api-server/conf/application.yaml`    |
+|Api Server| `api-server/conf/application.yaml`|
 
 The default configuration is as follows:
 
@@ -364,8 +358,7 @@ The default configuration is as follows:
 | spring.quartz.properties.org.quartz.jobStore.driverDelegateClass        | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate |
 | spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval     | 5000                                            |
 
-The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool
+The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool configuration is different.
-configuration is different.
 
 The default quartz threadpool configuration in *Master Server* is as follows:
 
@@ -376,8 +369,7 @@ The default quartz threadpool configuration in *Master Server* is as follows:
 | spring.quartz.properties.org.quartz.threadPool.threadPriority     | 5                                 |
 | spring.quartz.properties.org.quartz.threadPool.class              | org.quartz.simpl.SimpleThreadPool |
 
-Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured
+Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured as `QuartzZeroSizeThreadPool` which has zero thread;
-as `QuartzZeroSizeThreadPool` which has zero thread;
 The default configuration is as follows:
 
 |                      Parameters                      |                             Default value                             |
@@ -386,8 +378,7 @@ The default configuration is as follows:
 
 ### dolphinscheduler_env.sh [load environment variables configs]
 
-When using shell to commit tasks, DolphinScheduler will export environment variables
+When using shell to commit tasks, DolphinScheduler will export environment variables from `bin/env/dolphinscheduler_env.sh`. The
-from `bin/env/dolphinscheduler_env.sh`. The
 mainly configuration including `JAVA_HOME` and other environment paths.
 
 ```bash
@@ -415,10 +406,9 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec
 
 ### Log related configuration
 
-|    Service    |           Configuration file            |
+|Service| Configuration file  |
-|---------------|-----------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/logback-spring.xml` |
+|Master Server | `master-server/conf/logback-spring.xml`|
-| Api Server    | `api-server/conf/logback-spring.xml`    |
+|Api Server| `api-server/conf/logback-spring.xml`|
-| Worker Server | `worker-server/conf/logback-spring.xml` |
+|Worker Server| `worker-server/conf/logback-spring.xml`|
-| Alert Server  | `alert-server/conf/logback-spring.xml`  |
+|Alert Server| `alert-server/conf/logback-spring.xml`|
-

docs/docs/en/contribute/development-environment-setup.md

@@ -153,7 +153,7 @@ The browser access address [http://localhost:5173](http://localhost:5173) can lo
 
 #### zookeeper
 
-Download [ZooKeeper](https://zookeeper.apache.org/releases.html), and extract it.
+Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it.
 
 - Create directory `zkData` and `zkLog`
 - Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper

docs/docs/en/contribute/frontend-development.md

@@ -33,7 +33,7 @@ Use the command line mode `cd`  enter the `dolphinscheduler-ui` project director
 > If `npm install` is very slow, you can set the taobao mirror
 
 ```
-npm config set registry http://registry.npmmirror.com/
+npm config set registry http://registry.npm.taobao.org/
 ```
 
 - Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend:

docs/docs/en/faq.md

@@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory
 sudo rm -rf node_modules
 ```
 
-​ 2, install node-sass through npmmirror.com
+​ 2, install node-sass through npm.taobao.org
 
 ```
 sudo npm uninstall node-sass
-sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
+sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
 ```
 
 3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md)

docs/docs/en/guide/installation/cluster.md

@@ -14,7 +14,7 @@ Configure all the configurations refer to [pseudo-cluster deployment](pseudo-clu
 
 ### Modify Configuration
 
-This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster.md), the following describes the parameters that must be modified:
+This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster .md), the following describes the parameters that must be modified:
 
 ```shell
 # ---------------------------------------------------------

docs/docs/en/guide/installation/kubernetes.md

@@ -14,15 +14,16 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
 
 ## Install DolphinScheduler
 
-```bash
+Please download the source code package `apache-dolphinscheduler-<version>-src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download)
-# Choose the corresponding version yourself
+
-export VERSION=3.2.1
+To publish the release name `dolphinscheduler` version, please execute the following commands:
-helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
+
-tar -xvf dolphinscheduler-helm-${VERSION}.tgz
+```
-cd dolphinscheduler-helm
+$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
-helm repo add bitnami https://charts.bitnami.com/bitnami
+$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
-helm dependency update .
+$ helm repo add bitnami https://charts.bitnami.com/bitnami
-helm install dolphinscheduler .
+$ helm dependency update .
+$ helm install dolphinscheduler . --set image.tag=<version>
 ```
 
 To publish the release name `dolphinscheduler` version to `test` namespace:

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -2,7 +2,7 @@
 
 The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.
 
-If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
+If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
 
 ## Preparation
 
@@ -123,6 +123,7 @@ export SPRING_DATASOURCE_PASSWORD={password}
 # DolphinScheduler server related configuration
 export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
 export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
 
 # Registry center configuration, determines the type and link of the registry center
 export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}

docs/docs/en/guide/installation/standalone.md

@@ -36,7 +36,7 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch
 
 ### Start or Stop Server
 
-The script `./bin/dolphinscheduler-daemon.sh` can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
+The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
 
 ```shell
 # Start Standalone Server

docs/docs/en/guide/metrics/metrics.md

@@ -91,11 +91,6 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
   - stop: the number of stopped workflow instances
   - failover: the number of workflow instance fail-overs
 
-### RPC Related Metrics
-
-- ds.rpc.client.sync.request.exception.count: (counter) the number of exceptions occurred in sync rpc requests
-- ds.rpc.client.sync.request.duration.time: (histogram) the time cost of sync rpc requests
-
 ### Master Server Metrics
 
 - ds.master.overload.count: (counter) the number of times the master overloaded

docs/docs/en/guide/monitor.md

@@ -16,12 +16,6 @@
 
 ![worker](../../../img/new_ui/dev/monitor/worker.png)
 
-### Alert Server
-
-- Mainly related to alert server information.
-
-![alert-server](../../../img/new_ui/dev/monitor/alert-server.png)
-
 ### Database
 
 - Mainly the health status of the DB.
@@ -32,17 +26,18 @@
 
 ### Statistics
 
-![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png)
+![statistics](../../../img/new_ui/dev/monitor/statistics.png)
 
-Shows the command list in the system. Data is from the `t_ds_command` table.
+|             **Parameter**              |                  **Description**                   |
-
+|----------------------------------------|----------------------------------------------------|
-![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png)
+| Number of commands wait to be executed | Statistics of the `t_ds_command` table data.       |
-
+| The number of failed commands          | Statistics of the `t_ds_error_command` table data. |
-Shows the failure command list in the system. Data is from the `t_ds_error_command` table.
+| Number of tasks wait to run            | Count the data of `task_queue` in the ZooKeeper.   |
+| Number of tasks wait to be killed      | Count the data of `task_kill` in the ZooKeeper.    |
 
 ### Audit Log
 
 The audit log provides information about who accesses the system and the operations made to the system and record related
 time, which strengthen the security of the system and maintenance.
 
-![audit-log](../../../img/new_ui/dev/monitor/audit-log.png)
+![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg)

docs/docs/en/guide/parameter/global.md

@@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case,
 
 ### Save the workflow and set global parameters
 
-Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it.
+Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it.
 
 ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)
 

docs/docs/en/guide/parameter/project-parameter.md

@@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project.
 
 ### Define project-level parameters
 
-On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below:
+On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below:
 
 ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)
 

docs/docs/en/guide/parameter/startup-parameter.md

@@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config
 
 ## Usage
 
-Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters.
+Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters.
 
 ## Example
 

docs/docs/en/guide/resource/configuration.md

@@ -2,8 +2,8 @@
 
 - You could use `Resource Center` to upload text files, UDFs and other task-related files.
 - You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html) etc.
-- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resource Center` without the need of an external `HDFS` system or `S3`.
+- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resouce Center` without the need of an external `HDFS` system or `S3`.
-- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resource Center`. In this way, you could operate remote files as if on your local machines.
+- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resouce Center`. In this way, you could operate remote files as if on your local machines.
 
 ## Use Local File System
 
@@ -28,37 +28,74 @@ The configuration you may need to change:
 
 ## connect AWS S3
 
-if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following:
+if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
 
 config the following fields
 
 ```properties
+......
 
 resource.storage.type=S3
+
+......
+
+resource.aws.access.key.id=aws_access_key_id
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.secret.access.key=aws_secret_access_key
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.region=us-west-2
+# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
+resource.aws.s3.bucket.name=dolphinscheduler
+# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
+resource.aws.s3.endpoint=
+
+......
 ```
 
-```yaml
+## Use HDFS or Remote Object Storage
-aws:
-    s3:
-        # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
-        # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
-        # InstanceProfileCredentialsProvider: use the IAM role to authenticate
-        credentials.provider.type: AWSStaticCredentialsProvider
-        access.key.id: <access.key.id>
-        access.key.secret: <access.key.secret>
-        region: <region>
-        bucket.name: <bucket.name>
-        endpoint: <endpoint>
 
-```
+After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`.
-
-## connect OSS S3
-
-if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
-
-config the following fields
 
 ```properties
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# user data local directory path, please make sure the directory exists and have read write permissions
+data.basedir.path=/tmp/dolphinscheduler
+
+# resource view suffixs
+#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
+
+# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
+resource.storage.type=LOCAL
+# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
+resource.storage.upload.base.path=/tmp/dolphinscheduler
+
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.access.key.id=minioadmin
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.secret.access.key=minioadmin
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.region=cn-north-1
+# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
+resource.aws.s3.bucket.name=dolphinscheduler
+# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
+resource.aws.s3.endpoint=http://localhost:9000
+
 # alibaba cloud access key id, required if you set resource.storage.type=OSS 
 resource.alibaba.cloud.access.key.id=<your-access-key-id>
 # alibaba cloud access key secret, required if you set resource.storage.type=OSS
@@ -70,24 +107,89 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
 
-```
+# alibaba cloud access key id, required if you set resource.storage.type=OBS
-
-## connect OBS S3
-
-if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
-
-config the following fields
-
-```properties
-# access key id, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.id=<your-access-key-id>
-# access key secret, required if you set resource.storage.type=OBS
+# alibaba cloud access key secret, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.secret=<your-access-key-secret>
 # oss bucket name, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
 
+# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
+resource.hdfs.root.user=hdfs
+# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
+resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
+
+# whether to startup kerberos
+hadoop.security.authentication.startup.state=false
+
+# java.security.krb5.conf path
+java.security.krb5.conf.path=/opt/krb5.conf
+
+# login user from keytab username
+login.user.keytab.username=hdfs-mycluster@ESZ.COM
+
+# login user from keytab path
+login.user.keytab.path=/opt/hdfs.headless.keytab
+
+# kerberos expire time, the unit is hour
+kerberos.expire.time=2
+
+
+# resourcemanager port, the default value is 8088 if not specified
+resource.manager.httpaddress.port=8088
+# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
+yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
+# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
+yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
+# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
+yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
+
+# datasource encryption enable
+datasource.encryption.enable=false
+
+# datasource encryption salt
+datasource.encryption.salt=!@#$%^&*
+
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
+
+#data-quality.error.output.path=/tmp/data-quality-error-data
+
+# Network IP gets priority, default inner outer
+
+# Whether hive SQL is executed in the same session
+support.hive.oneSession=false
+
+# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
+sudo.enable=true
+
+# network interface preferred like eth0, default: empty
+#dolphin.scheduler.network.interface.preferred=
+
+# network IP gets priority, default: inner outer
+#dolphin.scheduler.network.priority.strategy=default
+
+# system env path
+#dolphinscheduler.env.path=dolphinscheduler_env.sh
+
+# development state
+development.state=false
+
+# rpc port
+alert.rpc.port=50052
+
+# set path of conda.sh
+conda.path=/opt/anaconda3/etc/profile.d/conda.sh
+
+# Task resource limit state
+task.resource.limit.state=false
+
+# way to collect applicationId: log(original regex match), aop
+appId.collect: log
 ```
 
 > **Note:**

docs/docs/en/guide/start/docker.md

@@ -128,7 +128,7 @@ and use `admin` and `dolphinscheduler123` as default username and password in th
 ![login](../../../../img/new_ui/dev/quick-start/login.png)
 
 > Note: If you start the services by the way [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper), and
-> starting with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running.
+> strating with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running.
 
 ## Change Environment Variable
 

docs/docs/en/guide/task/datafactory.md

@@ -19,11 +19,11 @@ DolphinScheduler DataFactory functions:
 
 ### Application Permission Setting
 
-First, visit the `Subscription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page.
+First, visit the `Subcription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page.
-![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
+![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
 After that, select `Contributor` role which satisfy functions calls in data factory. Then click `Members` page, and click `Select members`.
 Search application name or application `Object ID` to assign `Contributor` role to application.
-![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
+![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
 
 ## Configurations
 

docs/docs/en/guide/task/dms.md

@@ -73,17 +73,14 @@ Parameters of restarting the task by interface
 
 ## Environment to prepare
 
-Some AWS configuration is required, modify a field in file `aws.yaml`
+Some AWS configuration is required, modify a field in file `common.properties`
 
 ```yaml
-dms:
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-  credentials.provider.type: AWSStaticCredentialsProvider
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  access.key.id: <access.key.id>
+resource.aws.region=<AWS REGION>
-  access.key.secret: <access.key.secret>
-  region: <region>
-  endpoint: <endpoint>
 ```
 

docs/docs/en/guide/task/kubernetes.md

@@ -26,7 +26,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the
 | Command           | The container execution command (yaml-style array), for example: ["printenv"]                                                                                                                                                                                                                  |
 | Args              | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"]                                                                                                                                                                                                 |
 | Custom label      | The customized labels for k8s Job.                                                                                                                                                                                                                                                             |
-| Node selector     | The label selectors for running k8s pod. Different value in value set should be separated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. |
+| Node selector     | The label selectors for running k8s pod. Different value in value set should be seperated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. |
 | Custom parameter  | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables.                                                                                                                                                                               |
 
 ## Task Example

docs/docs/en/guide/task/mlflow.md

@@ -148,7 +148,7 @@ After this, you can visit the MLflow service (`http://localhost:5000`) page to v
 
 ### Preset Algorithm Repository Configuration
 
-If you can't access github, you can modify the following fields in the `common.properties` configuration file to replace the github address with an accessible address.
+If you can't access github, you can modify the following fields in the `commom.properties` configuration file to replace the github address with an accessible address.
 
 ```yaml
 # mlflow task plugin preset repository

docs/docs/en/guide/task/sagemaker.md

@@ -35,17 +35,14 @@ The task plugin are shown as follows:
 
 ## Environment to prepare
 
-Some AWS configuration is required, modify a field in file `aws.yaml`
+Some AWS configuration is required, modify a field in file `common.properties`
 
 ```yaml
-sagemaker:
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-  credentials.provider.type: AWSStaticCredentialsProvider
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  access.key.id: <access.key.id>
+resource.aws.region=<AWS REGION>
-  access.key.secret: <access.key.secret>
-  region: <region>
-  endpoint: <endpoint>
 ```
 

docs/docs/en/guide/task/spark.md

@@ -24,7 +24,6 @@ Spark task type for executing Spark application. When executing the Spark task,
 |----------------------------|------------------------------------------------------------------------------------------------------------------------------------|
 | Program type               | Supports Java, Scala, Python, and SQL.                                                                                             |
 | The class of main function | The **full path** of Main Class, the entry point of the Spark program.                                                             |
-| Master                     | The The master URL for the cluster.                                                                                                |
 | Main jar package           | The Spark jar package (upload by Resource Center).                                                                                 |
 | SQL scripts                | SQL statements in .sql files that Spark sql runs.                                                                                  |
 | Deployment mode            | <ul><li>spark submit supports three modes: cluster, client and local.</li><li>spark sql supports client and local modes.</li></ul> |

docs/docs/zh/architecture/configuration.md

@@ -130,44 +130,42 @@ export DOLPHINSCHEDULER_OPTS="
 
 > 不建议设置"-XX:DisableExplicitGC" , DolphinScheduler使用Netty进行通讯,设置该参数,可能会导致内存泄漏.
 >
->> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true"
+>> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true"将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6.
->> 将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6.
 
 ## 数据库连接相关配置
 
 在DolphinScheduler中使用Spring Hikari对数据库连接进行管理，配置文件位置：
 
-|     服务名称      |                 配置文件                  |
+|服务名称| 配置文件 |
-|---------------|---------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/application.yaml` |
+|Master Server | `master-server/conf/application.yaml`|
-| Api Server    | `api-server/conf/application.yaml`    |
+|Api Server| `api-server/conf/application.yaml`|
-| Worker Server | `worker-server/conf/application.yaml` |
+|Worker Server| `worker-server/conf/application.yaml`|
-| Alert Server  | `alert-server/conf/application.yaml`  |
+|Alert Server| `alert-server/conf/application.yaml`|
 
 默认配置如下：
 
-|                          参数                          |                        默认值                        |       描述        |
+|参数 | 默认值| 描述|
-|------------------------------------------------------|---------------------------------------------------|-----------------|
+|--|--|--|
-| spring.datasource.driver-class-name                  | org.postgresql.Driver                             | 数据库驱动           |
+|spring.datasource.driver-class-name| org.postgresql.Driver |数据库驱动|
-| spring.datasource.url                                | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | 数据库连接地址         |
+|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |数据库连接地址|
-| spring.datasource.username                           | root                                              | 数据库用户名          |
+|spring.datasource.username|root|数据库用户名|
-| spring.datasource.password                           | root                                              | 数据库密码           |
+|spring.datasource.password|root|数据库密码|
-| spring.datasource.hikari.connection-test-query       | select 1                                          | 检测连接是否有效的sql    |
+|spring.datasource.hikari.connection-test-query|select 1|检测连接是否有效的sql|
-| spring.datasource.hikari.minimum-idle                | 5                                                 | 最小空闲连接池数量       |
+|spring.datasource.hikari.minimum-idle| 5|最小空闲连接池数量|
-| spring.datasource.hikari.auto-commit                 | true                                              | 是否自动提交          |
+|spring.datasource.hikari.auto-commit|true|是否自动提交|
-| spring.datasource.hikari.pool-name                   | DolphinScheduler                                  | 连接池名称           |
+|spring.datasource.hikari.pool-name|DolphinScheduler|连接池名称|
-| spring.datasource.hikari.maximum-pool-size           | 50                                                | 连接池最大连接数        |
+|spring.datasource.hikari.maximum-pool-size|50|连接池最大连接数|
-| spring.datasource.hikari.connection-timeout          | 30000                                             | 连接超时时长          |
+|spring.datasource.hikari.connection-timeout|30000|连接超时时长|
-| spring.datasource.hikari.idle-timeout                | 600000                                            | 空闲连接存活最大时间      |
+|spring.datasource.hikari.idle-timeout|600000|空闲连接存活最大时间|
-| spring.datasource.hikari.leak-detection-threshold    | 0                                                 | 连接泄露检测阈值        |
+|spring.datasource.hikari.leak-detection-threshold|0|连接泄露检测阈值|
-| spring.datasource.hikari.initialization-fail-timeout | 1                                                 | 连接池初始化失败timeout |
+|spring.datasource.hikari.initialization-fail-timeout|1|连接池初始化失败timeout|
 
-DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名,
+DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名, 设置值即可。
-设置值即可。
 
-## 注册中心相关配置
+## Zookeeper相关配置
 
-DolphinScheduler默认使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置：
+DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置：
 |服务名称| 配置文件 |
 |--|--|
 |Master Server | `master-server/conf/application.yaml`|
@@ -176,23 +174,20 @@ DolphinScheduler默认使用Zookeeper进行集群管理、容错、事件监听
 
 默认配置如下：
 
-|                       参数                        |       默认值        |                                                                             描述                                                                             |
+|参数 |默认值| 描述|
-|-------------------------------------------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|--|--|--|
-| registry.zookeeper.namespace                    | dolphinscheduler | Zookeeper集群使用的namespace                                                                                                                                    |
+|registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace|
-| registry.zookeeper.connect-string               | localhost:2181   | Zookeeper集群连接信息                                                                                                                                            |
+|registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息|
-| registry.zookeeper.retry-policy.base-sleep-time | 60ms             | 基本重试时间差                                                                                                                                                    |
+|registry.zookeeper.retry-policy.base-sleep-time|60ms|基本重试时间差|
-| registry.zookeeper.retry-policy.max-sleep       | 300ms            | 最大重试时间                                                                                                                                                     |
+|registry.zookeeper.retry-policy.max-sleep|300ms|最大重试时间|
-| registry.zookeeper.retry-policy.max-retries     | 5                | 最大重试次数                                                                                                                                                     |
+|registry.zookeeper.retry-policy.max-retries|5|最大重试次数|
-| registry.zookeeper.session-timeout              | 30s              | session超时时间                                                                                                                                                |
+|registry.zookeeper.session-timeout|30s|session超时时间|
-| registry.zookeeper.connection-timeout           | 30s              | 连接超时时间                                                                                                                                                     |
+|registry.zookeeper.connection-timeout|30s|连接超时时间|
-| registry.zookeeper.block-until-connected        | 600ms            | 阻塞直到连接成功的等待时间                                                                                                                                              |
+|registry.zookeeper.block-until-connected|600ms|阻塞直到连接成功的等待时间|
-| registry.zookeeper.digest                       | {用户名:密码}         | 如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档) |
+|registry.zookeeper.digest|{用户名:密码}|如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档)|
 
 DolphinScheduler同样可以通过`bin/env/dolphinscheduler_env.sh`进行Zookeeper相关的配置。
 
-如果使用etcd作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md)。
-如果使用jdbc作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md)。
-
 ## common.properties [hadoop、s3、yarn配置]
 
 common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId收集相关的配置，配置文件位置：
@@ -205,8 +200,8 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 
 默认配置如下：
 
-|                      参数                       |                       默认值                        | 描述                                                                                                                                                                                                                   |
+|                      参数                       |                       默认值                        |                                                                                                          描述                                                                                                          |
-|-----------------------------------------------|--------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|-----------------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | data.basedir.path                             | /tmp/dolphinscheduler                            | 本地工作目录,用于存放临时文件                                                                                                                                                                                                      |
 | resource.storage.type                         | NONE                                             | 资源文件存储类型: HDFS,S3,OSS,GCS,ABS,NONE                                                                                                                                                                                   |
 | resource.upload.path                          | /dolphinscheduler                                | 资源文件存储路径                                                                                                                                                                                                             |
@@ -284,54 +279,48 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 
 位置：`master-server/conf/application.yaml`
 
-|                                     参数                                      |      默认值      |                                                                    描述                                                                    |
+|                           参数                           |     默认值      |                                        描述                                         |
-|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------|
+|--------------------------------------------------------|--------------|-----------------------------------------------------------------------------------|
-| master.listen-port                                                          | 5678          | master监听端口                                                                                                                               |
+| master.listen-port                                     | 5678         | master监听端口                                                                        |
-| master.pre-exec-threads                                                     | 10            | master准备执行任务的数量，用于限制并行的command                                                                                                           |
+| master.fetch-command-num                               | 10           | master拉取command数量                                                                 |
-| master.exec-threads                                                         | 100           | master工作线程数量,用于限制并行的流程实例数量                                                                                                               |
+| master.pre-exec-threads                                | 10           | master准备执行任务的数量，用于限制并行的command                                                    |
-| master.dispatch-task-number                                                 | 3             | master每个批次的派发任务数量                                                                                                                        |
+| master.exec-threads                                    | 100          | master工作线程数量,用于限制并行的流程实例数量                                                        |
-| master.host-selector                                                        | lower_weight  | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight                                                                  |
+| master.dispatch-task-number                            | 3            | master每个批次的派发任务数量                                                                 |
-| master.max-heartbeat-interval                                               | 10s           | master最大心跳间隔                                                                                                                             |
+| master.host-selector                                   | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight           |
-| master.task-commit-retry-times                                              | 5             | 任务重试次数                                                                                                                                   |
+| master.max-heartbeat-interval                          | 10s          | master最大心跳间隔                                                                      |
-| master.task-commit-interval                                                 | 1000          | 任务提交间隔,单位为毫秒                                                                                                                             |
+| master.task-commit-retry-times                         | 5            | 任务重试次数                                                                            |
-| master.state-wheel-interval                                                 | 5             | 轮询检查状态时间                                                                                                                                 |
+| master.task-commit-interval                            | 1000         | 任务提交间隔,单位为毫秒                                                                      |
-| master.server-load-protection.enabled                                       | true          | 是否开启系统保护策略                                                                                                                               |
+| master.state-wheel-interval                            | 5            | 轮询检查状态时间                                                                          |
-| master.server-load-protection.max-system-cpu-usage-percentage-thresholds    | 0.7           | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU                                                        |
+| master.max-cpu-load-avg                                | 1            | master最大cpuload均值,只有高于系统cpuload均值时,master服务才能调度任务. 默认值为1: 会使用100%的CPU             |
-| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds       | 0.7           | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU                                                  |
+| master.reserved-memory                                 | 0.3          | master预留内存,只有低于系统可用内存时,master服务才能调度任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流              |
-| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7           | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存                                                           |
+| master.failover-interval                               | 10           | failover间隔，单位为分钟                                                                  |
-| master.server-load-protection.max-disk-usage-percentage-thresholds          | 0.7           | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间                                                          |
+| master.kill-application-when-task-failover             | true         | 当任务实例failover时，是否kill掉yarn或k8s application                                        |
-| master.failover-interval                                                    | 10            | failover间隔，单位为分钟                                                                                                                         |
+| master.registry-disconnect-strategy.strategy           | stop         | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting                           |
-| master.kill-application-when-task-failover                                  | true          | 当任务实例failover时，是否kill掉yarn或k8s application                                                                                               |
+| master.registry-disconnect-strategy.max-waiting-time   | 100s         | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, |
-| master.registry-disconnect-strategy.strategy                                | stop          | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting                                                                                  |
+| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
-| master.registry-disconnect-strategy.max-waiting-time                        | 100s          | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
+| master.master.worker-group-refresh-interval            | 10s          | 定期将workerGroup从数据库中同步到内存的时间间隔                                                     |
-| master.master.worker-group-refresh-interval                                 | 10s           | 定期将workerGroup从数据库中同步到内存的时间间隔                                                                                                            |
-| master.command-fetch-strategy.type                                          | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED`                                                                                                       |
-| master.command-fetch-strategy.config.id-step                                | 1             | 数据库中t_ds_command的id自增步长                                                                                                                  |
-| master.command-fetch-strategy.config.fetch-size                             | 10            | master拉取command数量                                                                                                                        |
 
 ## Worker Server相关配置
 
 位置：`worker-server/conf/application.yaml`
 
-|                                     参数                                      |    默认值    |                                                                    描述                                                                     |
+|                          参数                          |    默认值    |                                                                    描述                                                                     |
-|-----------------------------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------|
+|------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------|
-| worker.listen-port                                                          | 1234      | worker监听端口                                                                                                                                |
+| worker.listen-port                                   | 1234      | worker监听端口                                                                                                                                |
-| worker.exec-threads                                                         | 100       | worker工作线程数量,用于限制并行的任务实例数量                                                                                                                |
+| worker.exec-threads                                  | 100       | worker工作线程数量,用于限制并行的任务实例数量                                                                                                                |
-| worker.max-heartbeat-interval                                               | 10s       | worker最大心跳间隔                                                                                                                              |
+| worker.max-heartbeat-interval                        | 10s       | worker最大心跳间隔                                                                                                                              |
-| worker.host-weight                                                          | 100       | 派发任务时，worker主机的权重                                                                                                                         |
+| worker.host-weight                                   | 100       | 派发任务时，worker主机的权重                                                                                                                         |
-| worker.tenant-auto-create                                                   | true      | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。                                                                 |
+| worker.tenant-auto-create                            | true      | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。                                                                 |
-| worker.server-load-protection.enabled                                       | true      | 是否开启系统保护策略                                                                                                                                |
+| worker.max-cpu-load-avg                              | 1         | worker最大cpuload均值,只有高于系统cpuload均值时,worker服务才能被派发任务. 默认值为1: 会使用100%的CPU                                                                    |
-| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds    | 0.7       | worker最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统CPU                                                         |
+| worker.reserved-memory                               | 0.3       | worker预留内存,只有低于系统可用内存时,worker服务才能被派发任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流                                                                     |
-| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds       | 0.7       | worker最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的JVM CPU                                                   |
+| worker.alert-listen-host                             | localhost | alert监听host                                                                                                                               |
-| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7       | worker最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统内存                                                            |
+| worker.alert-listen-port                             | 50052     | alert监听端口                                                                                                                                 |
-| worker.server-load-protection.max-disk-usage-percentage-thresholds          | 0.7       | worker最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统磁盘空间                                                           |
+| worker.registry-disconnect-strategy.strategy         | stop      | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting                                                                                   |
-| worker.alert-listen-host                                                    | localhost | alert监听host                                                                                                                               |
+| worker.registry-disconnect-strategy.max-waiting-time | 100s      | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
-| worker.alert-listen-port                                                    | 50052     | alert监听端口                                                                                                                                 |
+| worker.task-execute-threads-full-policy              | REJECT    | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务         |
-| worker.registry-disconnect-strategy.strategy                                | stop      | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting                                                                                   |
+| worker.tenant-config.auto-create-tenant-enabled      | true      | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。                                                                 |
-| worker.registry-disconnect-strategy.max-waiting-time                        | 100s      | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
+| worker.tenant-config.distributed-tenant-enabled      | false     | 如果设置为true, auto-create-tenant-enabled 将会不起作用。                                                                                             |
-| worker.task-execute-threads-full-policy                                     | REJECT    | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务         |
+| worker.tenant-config.default-tenant-enabled          | false     | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。                                                                                               |
-| worker.tenant-config.auto-create-tenant-enabled                             | true      | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。                                                                 |
-| worker.tenant-config.default-tenant-enabled                                 | false     | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。                                                                                               |
 
 ## Alert Server相关配置
 
@@ -377,9 +366,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 | spring.quartz.properties.org.quartz.threadPool.threadPriority     | 5                                 |
 | spring.quartz.properties.org.quartz.threadPool.class              | org.quartz.simpl.SimpleThreadPool |
 
-因为*Api Server*不会启动*Quartz Scheduler*
+因为*Api Server*不会启动*Quartz Scheduler*实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`不会启动任何线程。具体的默认配置如下：
-实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`
-不会启动任何线程。具体的默认配置如下：
 
 |                      Parameters                      |                             Default value                             |
 |------------------------------------------------------|-----------------------------------------------------------------------|
@@ -387,8 +374,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 
 ## dolphinscheduler_env.sh [环境变量配置]
 
-通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME`
+通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME` 任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。
-任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。
 
 ```bash
 # JAVA_HOME, will use it to start DolphinScheduler server
@@ -415,10 +401,9 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec
 
 ## 日志相关配置
 
-|     服务名称      |                  配置文件                   |
+|服务名称| 配置文件 |
-|---------------|-----------------------------------------|
+|--|--|
-| Master Server | `master-server/conf/logback-spring.xml` |
+|Master Server | `master-server/conf/logback-spring.xml`|
-| Api Server    | `api-server/conf/logback-spring.xml`    |
+|Api Server| `api-server/conf/logback-spring.xml`|
-| Worker Server | `worker-server/conf/logback-spring.xml` |
+|Worker Server| `worker-server/conf/logback-spring.xml`|
-| Alert Server  | `alert-server/conf/logback-spring.xml`  |
+|Alert Server| `alert-server/conf/logback-spring.xml`|
-

docs/docs/zh/contribute/development-environment-setup.md

@@ -148,7 +148,7 @@ pnpm run dev
 
 #### zookeeper
 
-下载 [ZooKeeper](https://zookeeper.apache.org/releases.html)，解压
+下载 [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3)，解压
 
 * 在 ZooKeeper 的目录下新建 zkData、zkLog文件夹
 * 将 conf 目录下的 `zoo_sample.cfg` 文件，复制一份，重命名为 `zoo.cfg`，修改其中数据和日志的配置，如：

docs/docs/zh/contribute/frontend-development.md

@@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2
 > 如果 `npm install` 速度非常慢，你可以设置淘宝镜像
 
 ```
-npm config set registry http://registry.npmmirror.com/
+npm config set registry http://registry.npm.taobao.org/
 ```
 
 - 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互：

docs/docs/zh/contribute/join/contribute.md

@@ -32,7 +32,7 @@
 如果你想实现某个 Feature 或者修复某个 Bug。请参考以下内容：
 
 * 所有的 Bug 与新 Feature 建议使用 Issues Page 进行管理。
-* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加到回复内容中。
+* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加的回复内容中。
 * 最好在核心贡献者找到一个导师（指导者），导师会在设计与功能实现上给予即时的反馈。
 * 你应该新建一个分支来开始你的工作，分支的名字参考[参与贡献 Pull Request 需知](./pull-request.md)。比如，你想完成 feature 功能并提交了 Issue 111，那么你的 branch 名字应为 feature-111。 功能名称可与导师讨论后确定。
 * 完成后，发送一个 Pull Request 到 dolphinscheduler，提交过程具体请参考下面《[提交代码流程](./submit-code.md)》。

docs/docs/zh/faq.md

@@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录
 sudo rm -rf node_modules
 ```
 
-​ 2，通过 npmmirror.com 下载 node-sass
+​ 2，通过 npm.taobao.org 下载 node-sass
 
 ```
 sudo npm uninstall node-sass
-sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
+sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
 ```
 
 3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md)

docs/docs/zh/guide/installation/kubernetes.md

@@ -14,15 +14,16 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务
 
 ## 安装 dolphinscheduler
 
-```bash
+请下载源码包 apache-dolphinscheduler-<version>-src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download)
-# 自行选择对应的版本
+
-export VERSION=3.2.1
+发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令：
-helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
+
-tar -xvf dolphinscheduler-helm-${VERSION}.tgz
+```
-cd dolphinscheduler-helm
+$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
-helm repo add bitnami https://charts.bitnami.com/bitnami
+$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
-helm dependency update .
+$ helm repo add bitnami https://charts.bitnami.com/bitnami
-helm install dolphinscheduler .
+$ helm dependency update .
+$ helm install dolphinscheduler . --set image.tag=<version>
 ```
 
 将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中：

docs/docs/zh/guide/installation/pseudo-cluster.md

@@ -118,6 +118,7 @@ export SPRING_DATASOURCE_PASSWORD={password}
 # DolphinScheduler server related configuration
 export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
 export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
 
 # Registry center configuration, determines the type and link of the registry center
 export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}

docs/docs/zh/guide/metrics/metrics.md

@@ -91,11 +91,6 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
   - stop：停止的工作流实例数量
   - failover：容错的工作流实例数量
 
-### RPC相关指标
-
-- ds.rpc.client.sync.request.exception.count: (counter) 同步rpc请求异常数
-- ds.rpc.client.sync.request.duration.time: (histogram) 同步rpc请求耗时
-
 ### Master Server指标
 
 - ds.master.overload.count: (counter) master过载次数

docs/docs/zh/guide/monitor.md

@@ -16,12 +16,6 @@
 
 ![worker](../../../img/new_ui/dev/monitor/worker.png)
 
-### Alert Server
-
-- 主要是 alert server 的相关信息。
-
-![alert-server](../../../img/new_ui/dev/monitor/alert-server.png)
-
 ### Database
 
 - 主要是 DB 的健康状况
@@ -32,16 +26,15 @@
 
 ### Statistics
 
-![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png)
+![statistics](../../../img/new_ui/dev/monitor/statistics.png)
 
-展示系统中的命令列表，数据来自`t_ds_command`表。
+- 待执行命令数：统计 t_ds_command 表的数据
-
+- 执行失败的命令数：统计 t_ds_error_command 表的数据
-![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png)
+- 待运行任务数：统计 Zookeeper 中 task_queue 的数据
-
+- 待杀死任务数：统计 Zookeeper 中 task_kill 的数据
-展示系统中的失败命令列表，数据来自`t_ds_error_command`表。
 
 ### 审计日志
 
-审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，对于维护安全都很有用。
+审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，他对于维护安全都很有用。
 
-![audit-log](../../../img/new_ui/dev/monitor/audit-log.png)
+![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg)

docs/docs/zh/guide/parameter/global.md

@@ -20,7 +20,7 @@
 
 ### 保存工作流，并设置全局参数
 
-全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示：
+全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示：
 
 ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)
 

docs/docs/zh/guide/parameter/project-parameter.md

@@ -8,7 +8,7 @@
 
 ### 定义项目级别参数
 
-在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示：
+在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示：
 
 ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)
 

docs/docs/zh/guide/parameter/startup-parameter.md

@@ -6,7 +6,7 @@
 
 ## 使用方式
 
-启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。
+启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。
 
 ## 任务样例
 

docs/docs/zh/guide/resource/configuration.md

@@ -26,35 +26,77 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
 
 ## 对接AWS S3
 
-如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下：
+如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
 
 配置以下字段
 
 ```properties
+......
 
 resource.storage.type=S3
+
+......
+
+resource.aws.access.key.id=aws_access_key_id
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.secret.access.key=aws_secret_access_key
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.region=us-west-2
+# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
+resource.aws.s3.bucket.name=dolphinscheduler
+# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
+resource.aws.s3.endpoint=
+
+......
 ```
 
-```yaml
+## 对接分布式或远端对象存储
-aws:
-    s3:
-        # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
-        # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
-        # InstanceProfileCredentialsProvider: use the IAM role to authenticate
-        credentials.provider.type: AWSStaticCredentialsProvider
-        access.key.id: <access.key.id>
-        access.key.secret: <access.key.secret>
-        region: <region>
-        bucket.name: <bucket.name>
-        endpoint: <endpoint>
 
-```
+当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置：
 
-## 对接阿里云 OSS
+### 配置 common.properties 文件
 
-如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
+在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
 
 ```properties
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# user data local directory path, please make sure the directory exists and have read write permissions
+data.basedir.path=/tmp/dolphinscheduler
+
+# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
+resource.storage.type=LOCAL
+
+# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
+# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
+resource.storage.upload.base.path=/tmp/dolphinscheduler
+
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.access.key.id=minioadmin
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.secret.access.key=minioadmin
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.region=cn-north-1
+# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
+resource.aws.s3.bucket.name=dolphinscheduler
+# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
+resource.aws.s3.endpoint=http://localhost:9000
+
 # alibaba cloud access key id, required if you set resource.storage.type=OSS 
 resource.alibaba.cloud.access.key.id=<your-access-key-id>
 # alibaba cloud access key secret, required if you set resource.storage.type=OSS
@@ -66,27 +108,92 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
 
-```
+# alibaba cloud access key id, required if you set resource.storage.type=OBS
-
-## 对接华为云 OBS
-
-如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
-
-```properties
-# access key id, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.id=<your-access-key-id>
-# access key secret, required if you set resource.storage.type=OBS
+# alibaba cloud access key secret, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.secret=<your-access-key-secret>
 # oss bucket name, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
 
+# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
+resource.hdfs.root.user=root
+# if resource.storage.type=S3, the value like: s3a://dolphinscheduler;
+# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
+resource.hdfs.fs.defaultFS=hdfs://localhost:8020
+
+# whether to startup kerberos
+hadoop.security.authentication.startup.state=false
+
+# java.security.krb5.conf path
+java.security.krb5.conf.path=/opt/krb5.conf
+
+# login user from keytab username
+login.user.keytab.username=hdfs-mycluster@ESZ.COM
+
+# login user from keytab path
+login.user.keytab.path=/opt/hdfs.headless.keytab
+
+# kerberos expire time, the unit is hour
+kerberos.expire.time=2
+# resource view suffixs
+#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
+
+# resourcemanager port, the default value is 8088 if not specified
+resource.manager.httpaddress.port=8088
+# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
+yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
+# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value;
+# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
+yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s
+# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
+yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s
+
+# datasource encryption enable
+datasource.encryption.enable=false
+
+# datasource encryption salt
+datasource.encryption.salt=!@#$%^&*
+
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
+
+#data-quality.error.output.path=/tmp/data-quality-error-data
+
+# Network IP gets priority, default inner outer
+
+# Whether hive SQL is executed in the same session
+support.hive.oneSession=false
+
+# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions;
+# if set false, executing user is the deploy user and doesn't need sudo permissions
+sudo.enable=true
+
+# network interface preferred like eth0, default: empty
+#dolphin.scheduler.network.interface.preferred=
+
+# network IP gets priority, default: inner outer
+#dolphin.scheduler.network.priority.strategy=default
+
+# system env path
+#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
+
+# development state
+development.state=false
+
+# rpc port
+alert.rpc.port=50052
+
+# way to collect applicationId: log(original regex match), aop
+appId.collect: log
 ```
 
 > **注意**：
 >
 > * 如果只配置了 `api-server/conf/common.properties` 的文件，则只是开启了资源上传的操作，并不能满足正常使用。如果想要在工作流中执行相关文件则需要额外配置 `worker-server/conf/common.properties`。
 > * 如果用到资源上传的功能，那么[安装部署](../installation/standalone.md)中，部署用户需要有这部分的操作权限。
-> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过此步骤。
+> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过次步骤。
 

docs/docs/zh/guide/task/appendix.md

@@ -8,7 +8,7 @@
 |----------|--------------------------------------------------------------------------------------------------------------------------------------|
 | 任务名称     | 任务的名称，同一个工作流定义中的节点名称不能重复。                                                                                                            |
 | 运行标志     | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。                                                                                                   |
-| 缓存执行     | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。                                              |
+| 缓存执行     | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。                                              |
 | 描述       | 当前节点的功能描述。                                                                                                                           |
 | 任务优先级    | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。                                                                                       |
 | Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。                                                                              |

docs/docs/zh/guide/task/datafactory.md

@@ -19,10 +19,10 @@ DolphinScheduler DataFactory 组件的功能:
 
 ### 应用权限设置
 
-首先打开当前`Subscription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。
+首先打开当前`Subcription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。
-![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
+![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
 首先选择`Contributor`角色足够满足调用数据工厂。然后选择`Members`页面，再选择`Select members`，检索APP名称或APP的`Object ID`并添加，从给指定APP添加权限.
-![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
+![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
 
 ## 环境配置
 

docs/docs/zh/guide/task/dms.md

@@ -73,17 +73,14 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS
 
 ## 环境配置
 
-需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
+需要进行AWS的一些配置，修改`common.properties`中的以下配置信息
 
 ```yaml
-dms:
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-  credentials.provider.type: AWSStaticCredentialsProvider
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  access.key.id: <access.key.id>
+resource.aws.region=<AWS REGION>
-  access.key.secret: <access.key.secret>
-  region: <region>
-  endpoint: <endpoint>
 ```
 

docs/docs/zh/guide/task/mlflow.md

@@ -139,7 +139,7 @@ mlflow server -h 0.0.0.0 -p 5000 --serve-artifacts --backend-store-uri sqlite://
 
 ### 内置算法仓库配置
 
-如果遇到github无法访问的情况，可以修改`common.properties`配置文件的以下字段，将github地址替换能访问的地址。
+如果遇到github无法访问的情况，可以修改`commom.properties`配置文件的以下字段，将github地址替换能访问的地址。
 
 ```yaml
 # mlflow task plugin preset repository

docs/docs/zh/guide/task/sagemaker.md

@@ -33,17 +33,14 @@ DolphinScheduler SageMaker 组件的功能:
 
 ## 环境配置
 
-需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
+需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息
 
 ```yaml
-sagemaker:
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-  credentials.provider.type: AWSStaticCredentialsProvider
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-  access.key.id: <access.key.id>
+resource.aws.region=<AWS REGION>
-  access.key.secret: <access.key.secret>
-  region: <region>
-  endpoint: <endpoint>
 ```
 

docs/docs/zh/guide/task/spark.md

@@ -23,7 +23,6 @@ Spark  任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支
 - 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。
 - 主函数的 Class：Spark 程序的入口 Main class 的全路径。
 - 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。
-- Master：执行 Spark 集群的 Master Url。
 - SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。
 - 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。
   (2) spark sql 支持 client 和 local 两种模式。