doc: Add hyperlink for pydolphinscheduler

(cherry picked from commit f937c511b1)

.dlc.json

@@ -3,37 +3,37 @@
     {
       "pattern": "^http://localhost"
     },
-    {
-      "pattern": "^https://hive.apache.org"
-    },
-    {
-      "pattern": "^http://192"
-    },
     {
       "pattern": "^https://img.shields.io/badge"
+    },
+    {
+      "pattern": "/community/community.html$"
+    },
+    {
+      "pattern": "^https://dolphinscheduler.apache.org"
     }
   ],
   "replacementPatterns": [
     {
       "pattern": "^/en-us/download/download.html$",
-      "replacement": "https://dolphinscheduler.apache.org/en-us/download/download.html"
+      "replacement": "https://dolphinscheduler.apache.org/en-us/download"
     },
     {
       "pattern": "^/zh-cn/download/download.html$",
-      "replacement": "https://dolphinscheduler.apache.org/zh-cn/download/download.html"
+      "replacement": "https://dolphinscheduler.apache.org/zh-cn/download"
-    },
+    }
+  ],
+  "httpHeaders": [
     {
-      "pattern": "^/img",
+      "urls": ["https://docs.github.com/"],
-      "replacement": "{{BASEURL}}/docs/img"
+      "headers": {
+        "Accept-Encoding": "zstd, br, gzip, deflate"
+      }
     }
   ],
   "timeout": "10s",
   "retryOn429": true,
   "retryCount": 10,
   "fallbackRetryDelay": "1000s",
-  "aliveStatusCodes": [
+  "aliveStatusCodes": [200, 401, 0]
-    200,
-    401,
-    0
-  ]
 }

.github/ISSUE_TEMPLATE/improvement-report.yml

@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Improvement request
+description: Suggest an improvement for this project
+title: "[Improvement][Module Name] Improvement title"
+labels: [ "improvement", "Waiting for reply" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        For better global communication, Please write in English.
+
+        If you feel the description in English is not clear, then you can append description in Chinese, thanks!
+
+  - type: checkboxes
+    attributes:
+      label: Search before asking
+      description: >
+        Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first
+        to see whether the same feature was requested already.
+      options:
+        - label: >
+            I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no
+            similar feature requirement.
+          required: true
+
+  - type: textarea
+    attributes:
+      label: Description
+      description: A short description why your want to do this improvement
+
+  - type: checkboxes
+    attributes:
+      label: Are you willing to submit a PR?
+      description: >
+        This is absolutely not required, but we are happy to guide you in the contribution process
+        especially if you already have a good understanding of how to implement the improvement.
+        DolphinScheduler is a totally community-driven project and we love to bring new contributors in.
+      options:
+        - label: Yes I am willing to submit a PR!
+
+  - type: checkboxes
+    attributes:
+      label: Code of Conduct
+      description: |
+        The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
+      options:
+        - label: |
+            I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
+          required: true
+
+  - type: markdown
+    attributes:
+      value: "Thanks for completing our form!"

.github/workflows/backend.yml

@@ -56,18 +56,26 @@ jobs:
     needs: paths-filter
     if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'adopt'
       - name: Sanity Check
         uses: ./.github/actions/sanity-check
       - uses: actions/cache@v2
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven
-      - name: Build and Package
+      - name: Build and Package on ${{ matrix.java }}
         run: |
           ./mvnw -B clean install \
                  -Prelease,docker \
@@ -78,11 +86,43 @@ jobs:
                  -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
       - name: Check dependency license
         run: tools/dependencies/check-LICENSE.sh
+      - uses: actions/upload-artifact@v2
+        if: ${{ matrix.java == '8' }}
+        name: Upload Binary Package
+        with:
+          name: binary-package-${{ matrix.java }}
+          path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
+          retention-days: 1
+  cluster-test:
+    name: ${{ matrix.case.name }}
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        case:
+          - name: cluster-test-mysql
+            script: .github/workflows/cluster-test/mysql/start-job.sh
+          - name: cluster-test-postgresql
+            script: .github/workflows/cluster-test/postgresql/start-job.sh
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: actions/download-artifact@v2
+        name: Download Binary Package
+        with:
+          # Only run cluster test on jdk8
+          name: binary-package-8
+          path: ./
+      - name: Running cluster test
+        run: |
+          /bin/bash ${{ matrix.case.script }}
   result:
     name: Build
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    needs: [ build, paths-filter ]
+    needs: [ build, paths-filter, cluster-test ]
     if: always()
     steps:
       - name: Status
@@ -91,7 +131,7 @@ jobs:
             echo "Skip Build!"
             exit 0
           fi
-          if [[ ${{ needs.build.result }} != 'success' ]]; then
+          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
             echo "Build Failed!"
             exit -1
           fi

.github/workflows/cluster-test/mysql/Dockerfile

@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+RUN apt update ; \
+    apt install -y curl wget default-mysql-client sudo openssh-server netcat-traditional ;
+
+COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
+RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+
+RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+
+ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+
+#Setting install.sh
+COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+
+#Setting dolphinscheduler_env.sh
+COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+
+#Download mysql jar
+ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+
+#Deploy
+COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
+
+CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/mysql/deploy.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euox pipefail
+
+
+USER=root
+
+#Create database
+mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
+
+#Sudo
+sed -i '$a'$USER'  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
+sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
+
+#SSH
+ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+service ssh start
+
+#Init schema
+/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
+
+#Start Cluster
+/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
+
+#Keep running
+tail -f /dev/null

.github/workflows/cluster-test/mysql/docker-compose-base.yaml

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  mysql:
+    container_name: mysql
+    image: mysql:5.7.36
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: 123456
+    ports:
+      - "3306:3306"
+    healthcheck:
+      test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  zoo1:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo2:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo2
+    ports:
+      - "2182:2181"
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo3:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo3
+    ports:
+      - "2183:2181"
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+

.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  ds:
+    container_name: ds
+    image: jdk8:ds_mysql_cluster
+    restart: always
+    ports:
+      - "12345:12345"
+      - "5679:5679"
+      - "1235:1235"
+      - "50053:50053"

.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh

@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/usr/local/openjdk-8}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
+export SPRING_DATASOURCE_USERNAME=root
+export SPRING_DATASOURCE_PASSWORD=123456
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

.github/workflows/cluster-test/mysql/install_env.sh

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ---------------------------------------------------------
+# INSTALL MACHINE
+# ---------------------------------------------------------
+# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
+# including master, worker, api, alert. If you want to deploy in pseudo-distributed
+# mode, just write a pseudo-distributed hostname
+# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
+ips=${ips:-"localhost"}
+
+# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
+# modify it if you use different ssh port
+sshPort=${sshPort:-"22"}
+
+# A comma separated list of machine hostname or IP would be installed Master server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
+masters=${masters:-"localhost"}
+
+# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
+# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
+# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
+workers=${workers:-"localhost:default"}
+
+# A comma separated list of machine hostname or IP would be installed Alert server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
+alertServer=${alertServer:-"localhost"}
+
+# A comma separated list of machine hostname or IP would be installed API server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
+apiServers=${apiServers:-"localhost"}
+
+# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
+# Do not set this configuration same as the current path (pwd)
+installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
+
+# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
+# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
+# to be created by this user
+deployUser=${deployUser:-"dolphinscheduler"}
+
+# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
+zkRoot=${zkRoot:-"/dolphinscheduler"}

.github/workflows/cluster-test/mysql/running_test.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+
+API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
+MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
+WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
+ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
+
+#Cluster start health check
+TIMEOUT=120
+START_HEALTHCHECK_EXITCODE=0
+
+for ((i=1; i<=TIMEOUT; i++))
+do
+  MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+  WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+  API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+  ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
+  if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
+    START_HEALTHCHECK_EXITCODE=0
+  else
+    START_HEALTHCHECK_EXITCODE=2
+  fi
+
+  if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
+    echo "cluster start health check success"
+    break
+  fi
+
+  if [[ $i -eq $TIMEOUT ]];then
+    docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
+    echo "cluster start health check failed"
+    exit $START_HEALTHCHECK_EXITCODE
+  fi
+
+  sleep 1
+done
+
+#Stop Cluster
+docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
+
+#Cluster stop health check
+sleep 5
+MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
+  echo "master stop health check success"
+else
+  echo "master stop health check failed"
+  exit 3
+fi
+
+WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
+  echo "worker stop health check success"
+else
+  echo "worker stop health check failed"
+  exit 3
+fi
+
+API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+if [[ $API_HTTP_STATUS -ne 200 ]];then
+  echo "api stop health check success"
+else
+  echo "api stop health check failed"
+  exit 3
+fi
+
+ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
+if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
+  echo "alert stop health check success"
+else
+  echo "alert stop health check failed"
+  exit 3
+fi

.github/workflows/cluster-test/mysql/start-job.sh

@@ -15,14 +15,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+set -euox pipefail
 
-BIN_DIR=$(dirname $0)
+#Start base service containers
-DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)}
+docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
 
-source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh"
+#Build ds mysql cluster image
+docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile .
 
-JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+#Start ds mysql cluster container
+docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
 
-java $JAVA_OPTS \
+#Running tests
-  -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \
+/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
-  org.apache.dolphinscheduler.tools.datasource.CreateDolphinScheduler
+
+#Cleanup
+docker rm -f $(docker ps -aq)

.github/workflows/cluster-test/postgresql/Dockerfile

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+RUN apt update ; \
+    apt install -y curl wget sudo openssh-server netcat-traditional ;
+
+COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
+RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+
+RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+
+ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+
+#Setting install.sh
+COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+
+#Setting dolphinscheduler_env.sh
+COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+
+#Deploy
+COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
+
+CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/postgresql/deploy.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euox pipefail
+
+
+USER=root
+
+#Sudo
+sed -i '$a'$USER'  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
+sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
+
+#SSH
+ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+service ssh start
+
+#Init schema
+/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
+
+#Start Cluster
+/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
+
+#Keep running
+tail -f /dev/null

.github/workflows/cluster-test/postgresql/docker-compose-base.yaml

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  postgres:
+    container_name: postgres
+    image: postgres:14.1
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: dolphinscheduler
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  zoo1:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo2:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo2
+    ports:
+      - "2182:2181"
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo3:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo3
+    ports:
+      - "2183:2181"
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+

.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  ds:
+    container_name: ds
+    image: jdk8:ds_postgresql_cluster
+    restart: always
+    ports:
+      - "12345:12345"
+      - "5679:5679"
+      - "1235:1235"
+      - "50053:50053"

.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh

@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/usr/local/openjdk-8}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
+export SPRING_DATASOURCE_USERNAME=postgres
+export SPRING_DATASOURCE_PASSWORD=postgres
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

.github/workflows/cluster-test/postgresql/install_env.sh

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ---------------------------------------------------------
+# INSTALL MACHINE
+# ---------------------------------------------------------
+# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
+# including master, worker, api, alert. If you want to deploy in pseudo-distributed
+# mode, just write a pseudo-distributed hostname
+# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
+ips=${ips:-"localhost"}
+
+# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
+# modify it if you use different ssh port
+sshPort=${sshPort:-"22"}
+
+# A comma separated list of machine hostname or IP would be installed Master server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
+masters=${masters:-"localhost"}
+
+# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
+# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
+# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
+workers=${workers:-"localhost:default"}
+
+# A comma separated list of machine hostname or IP would be installed Alert server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
+alertServer=${alertServer:-"localhost"}
+
+# A comma separated list of machine hostname or IP would be installed API server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
+apiServers=${apiServers:-"localhost"}
+
+# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
+# Do not set this configuration same as the current path (pwd)
+installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
+
+# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
+# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
+# to be created by this user
+deployUser=${deployUser:-"dolphinscheduler"}
+
+# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
+zkRoot=${zkRoot:-"/dolphinscheduler"}

.github/workflows/cluster-test/postgresql/running_test.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+
+API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
+MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
+WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
+ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
+
+#Cluster start health check
+TIMEOUT=120
+START_HEALTHCHECK_EXITCODE=0
+
+for ((i=1; i<=TIMEOUT; i++))
+do
+  MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+  WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+  API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+  ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
+  if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
+    START_HEALTHCHECK_EXITCODE=0
+  else
+    START_HEALTHCHECK_EXITCODE=2
+  fi
+
+  if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
+    echo "cluster start health check success"
+    break
+  fi
+
+  if [[ $i -eq $TIMEOUT ]];then
+    docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
+    echo "cluster start health check failed"
+    exit $START_HEALTHCHECK_EXITCODE
+  fi
+
+  sleep 1
+done
+
+#Stop Cluster
+docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
+
+#Cluster stop health check
+sleep 5
+MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
+  echo "master stop health check success"
+else
+  echo "master stop health check failed"
+  exit 3
+fi
+
+WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
+  echo "worker stop health check success"
+else
+  echo "worker stop health check failed"
+  exit 3
+fi
+
+API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+if [[ $API_HTTP_STATUS -ne 200 ]];then
+  echo "api stop health check success"
+else
+  echo "api stop health check failed"
+  exit 3
+fi
+
+ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
+if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
+  echo "alert stop health check success"
+else
+  echo "alert stop health check failed"
+  exit 3
+fi

.github/workflows/cluster-test/postgresql/start-job.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euox pipefail
+
+#Start base service containers
+docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
+
+#Build ds postgresql cluster image
+docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile .
+
+#Start ds postgresql cluster container
+docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
+
+#Running tests
+/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
+
+#Cleanup
+docker rm -f $(docker ps -aq)

.github/workflows/publish-docker.yaml

@@ -20,9 +20,9 @@ on:
   push:
     branches:
       - dev
-
+  release:
-env:
+    types:
-  HUB: ghcr.io/apache/dolphinscheduler
+      - released
 
 jobs:
   build:
@@ -41,16 +41,29 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+      - name: Set environment variables
+        run: |
+          if [[ ${{ github.event_name }} == "release" ]]; then
+            echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV
+            echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV
+            echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV
+            echo "HUB=apache" >> $GITHUB_ENV
+          else
+            echo "DOCKER_REGISTRY=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
+            echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV
+            echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
+            echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
+          fi
       - name: Log in to the Container registry
-        uses: docker/login-action@v1.10.0
+        uses: docker/login-action@v2
         with:
-          registry: ${{ env.HUB }}
+          registry: ${{ env.DOCKER_REGISTRY }}
-          username: ${{ github.actor }}
+          username: ${{ env.DOCKER_USERNAME }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          password: ${{ env.DOCKER_PASSWORD }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Build and push docker images
         run: |
           ./mvnw -B clean deploy \

.github/workflows/publish-helm-chart.yaml

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: publish-helm-chart
+
+on:
+  push:
+    branches:
+      - dev
+  release:
+    types:
+      - released
+
+env:
+  HUB: ghcr.io/apache/dolphinscheduler
+
+jobs:
+  build:
+    if: github.repository == 'apache/dolphinscheduler'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set environment variables
+        run: |
+          # TODO
+          if [[ ${{ github.event_name }} == "release" ]]; then
+            echo "HUB=registry-1.docker.io/apache" >> $GITHUB_ENV
+            echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV
+            echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV
+            echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV
+          else
+            echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
+            echo "DOCKER_REGISTRY=c" >> $GITHUB_ENV
+            echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV
+            echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
+          fi
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+      - name: Publish Helm Chart
+        working-directory: deploy/kubernetes
+        run: |
+          if [[ ${{ env.HUB }} == "ghcr.io/apache/dolphinscheduler" ]]; then
+            VERSION=0.0.0-$(git rev-parse --short HEAD)
+            sed -i "s/^version: .*/version: $VERSION/" dolphinscheduler/Chart.yaml
+          fi
+          helm dep up dolphinscheduler
+          helm package dolphinscheduler
+          helm push dolphinscheduler-helm-*.tgz oci://${{ env.HUB }}

.github/workflows/py-ci.yml

@@ -22,8 +22,6 @@ on:
     paths:
       - 'dolphinscheduler-python/**'
   pull_request:
-    paths:
-      - 'dolphinscheduler-python/**'
 
 concurrency:
   group: py-${{ github.event.pull_request.number || github.ref }}
@@ -38,14 +36,32 @@ env:
   DEPENDENCES: pip setuptools wheel tox
 
 jobs:
+  paths-filter:
+    name: Python-Path-Filter
+    runs-on: ubuntu-latest
+    outputs:
+      not-docs: ${{ steps.filter.outputs.not-docs }}
+      py-change: ${{ steps.filter.outputs.py-change }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
+        id: filter
+        with:
+          filters: |
+            not-docs:
+              - '!(docs/**)'
+            py-change:
+              - 'dolphinscheduler-python/pydolphinscheduler/**'
   lint:
     name: Lint
+    if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }}
     timeout-minutes: 15
+    needs: paths-filter
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install Dependences
@@ -62,12 +78,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        # YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also:
-        os: [ubuntu-18.04, macOS-latest, windows-latest]
+        # https://github.com/actions/setup-python/issues/160#issuecomment-724485470
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev]
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        # Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments
+        exclude:
+          - os: windows-latest
+            python-version: '3.10'
+          - os: windows-latest
+            python-version: 3.11-dev
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependences
@@ -81,18 +105,22 @@ jobs:
     timeout-minutes: 15
     needs: lint
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env-list: [doc-build, doc-build-multi]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install Dependences
         run: |
           python -m pip install --upgrade ${{ env.DEPENDENCES }}
-      - name: Run Tests Build Docs
+      - name: Run Build Docs Tests ${{ matrix.env-list }}
         run: |
-          python -m tox -vv -e doc-build-test
+          python -m tox -vv -e ${{ matrix.env-list }}
   local-ci:
     name: Local CI
     timeout-minutes: 15
@@ -103,7 +131,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install Dependences
@@ -112,14 +140,12 @@ jobs:
       - name: Run Tests Build Docs
         run: |
           python -m tox -vv -e local-ci
-  build-image:
+  integrate-test:
-    name: Build Image
+    name: Integrate Test
+    if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }}
     runs-on: ubuntu-latest
-    # Switch to project root directory to run mvnw command
+    needs: paths-filter
-    defaults:
+    timeout-minutes: 30
-      run:
-        working-directory: ./
-    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
         with:
@@ -132,7 +158,9 @@ jobs:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-maven-
+      # Switch to project root directory to run mvnw command
       - name: Build Image
+        working-directory: ./
         run: |
           ./mvnw -B clean install \
           -Dmaven.test.skip \
@@ -140,39 +168,36 @@ jobs:
           -Dmaven.checkstyle.skip \
           -Pdocker,release -Ddocker.tag=ci \
           -pl dolphinscheduler-standalone-server -am
-      - name: Export Docker Images
-        run: |
-          docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
-          && du -sh /tmp/standalone-image.tar
-      - uses: actions/upload-artifact@v2
-        name: Upload Docker Images
-        with:
-          name: standalone-image
-          path: /tmp/standalone-image.tar
-          retention-days: 1
-  integrate-test:
-    name: Integrate Test
-    timeout-minutes: 20
-    needs:
-      - build-image
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        name: Download Docker Images
-        with:
-          name: standalone-image
-          path: /tmp
-      - name: Load Docker Images
-        run: |
-          docker load -i /tmp/standalone-image.tar
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install Dependences
         run: |
           python -m pip install --upgrade ${{ env.DEPENDENCES }}
-      - name: Run Tests Build Docs
+      - name: Run Integrate Tests
         run: |
           python -m tox -vv -e integrate-test
+  result:
+    name: Python
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [ paths-filter, local-ci, integrate-test ]
+    if: always()
+    steps:
+      - name: Status
+        # We need change CWD to current directory to avoid global default working directory not exists
+        working-directory: ./
+        run: |
+          if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "Only document change, skip both python unit and integrate test!"
+            exit 0
+          fi
+          if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "No python code change, and integrate test pass!"
+            exit 0
+          fi
+          if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then
+            echo "py-ci Failed!"
+            exit -1
+          fi

.github/workflows/unit-test.yml

@@ -53,17 +53,20 @@ jobs:
     needs: paths-filter
     if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    strategy:
+      matrix:
+        java: ['8', '11']
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: true
       - name: Sanity Check
         uses: ./.github/actions/sanity-check
-      - name: Set up JDK 1.8
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v2
         with:
-          java-version: 8
+          java-version: ${{ matrix.java }}
           distribution: 'adopt'
       - uses: actions/cache@v2
         with:

.gitignore

@@ -6,6 +6,7 @@
 .DS_Store
 .target
 .idea/
+.run/
 target/
 dist/
 all-dependencies.txt

CONTRIBUTING.md

@@ -1,101 +1 @@
-# How To Contribute
+Please refer to the contribution document [How to contribute](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/contribute/join/contribute.html)
-
-Start by forking the dolphinscheduler GitHub repository, make changes in a branch and then send a pull request.
-
-## Set up your dolphinscheduler GitHub Repository
-
-There are three branches in the remote repository currently:
-
-  - `master` : normal delivery branch. After the stable version is released, the code for the stable version branch is merged into the master branch.
-            
-  - `dev` : daily development branch. The daily development branch, the newly submitted code can pull requests to this branch.
-  
-  - `x.x.x-release` : the stable release version.
-
-So, you should fork the `dev` branch.
-
-After forking the [dolphinscheduler upstream source repository](https://github.com/apache/dolphinscheduler/fork) to your personal repository, you can set your  personal development environment.
-
-```sh
-cd <your work direcotry>
-git clone <your personal forked dolphinscheduler repo>
-cd dolphinscheduler
-```
-
-## Set git remote as `upstream`
-
-Add remote repository address, named upstream
-
-```sh
-git remote add upstream https://github.com/apache/dolphinscheduler.git
-```
-
-View repository:
-
-```sh
-git remote -v
-```
-
-There will be two repositories at this time: origin (your own warehouse) and upstream (remote repository)
-
-Get/update remote repository code (already the latest code, skip it).
-
-
-```sh
-git fetch upstream
-```
-
-Synchronize remote repository code to local repository
-
-```sh
-git checkout origin/dev
-git merge --no-ff upstream/dev
-```
-
-If remote branch has a new branch `dev-1.0`, you need to synchronize this branch to the local repository, then push to your own repository.
-
-```sh
-git checkout -b dev-1.0 upstream/dev-1.0
-git push --set-upstream origin dev-1.0
-```
-
-## Create your feature branch
-
-Before making code changes, make sure you create a separate branch for them.
-
-```sh
-git checkout -b <your-feature-branch> dev
-```
-
-## Commit changes
-
-After modifying the code locally, submit it to your own repository:
-
-```sh
-git commit -m 'information about your feature'
-```
-
-## Push to the branch
-
-Push your locally committed changes to the remote origin (your fork).
-
-```sh
-git push origin <your-feature-branch>
-```
-
-## Create a pull request
-
-After submitting changes to your remote repository, you should click on the new pull request On the following github page.
-
-<p align = "center">
-<img src = "http://geek.analysys.cn/static/upload/221/2019-04-02/90f3abbf-70ef-4334-b8d6-9014c9cf4c7f.png" width ="60%"/>
-</p>
-
-
-Select the modified local branch and the branch to merge past to create a pull request.
-
-<p align = "center">
-<img src = "http://geek.analysys.cn/static/upload/221/2019-04-02/fe7eecfe-2720-4736-951b-b3387cf1ae41.png" width ="60%"/>
-</p>
-
-Next, the administrator is responsible for **merging** to complete the pull request.

README.md

@@ -15,61 +15,51 @@ Dolphin Scheduler Official Website
 [![Stargazers over time](https://starchart.cc/apache/dolphinscheduler.svg)](https://starchart.cc/apache/dolphinscheduler)
 
 [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
-[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
 
-## Design Features
+## Features
 
-DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`.
+Apache DolphinScheduler is the modern data workflow orchestration platform with powerful user interface, dedicated to solving complex task dependencies in the data pipeline and providing various types of jobs available `out of the box`
 
-Its main objectives are as follows:
+The key features for DolphinScheduler are as follows:
- -  Highly Reliable, 
+- Easy to deploy, we provide 4 ways to deploy, such as Standalone deployment,Cluster deployment,Docker / Kubernetes deployment and Rainbond deployment
-DolphinScheduler adopts a decentralized multi-master and multi-worker architecture design, which naturally supports easy expansion and high availability (not restricted by a single point of bottleneck), and its performance increases linearly with the increase of machines
+- Easy to use, there are 3 ways to create workflows:
- - High performance, supporting tens of millions of tasks every day
+  - Visually, create tasks by dragging and dropping tasks
- - Support multi-tenant.
+  - Creating workflows by PyDolphinScheduler(Python way)
- - Cloud Native, DolphinScheduler supports multi-cloud/data center workflow management, also
+  - Creating workflows through Open API
-supports Kubernetes, Docker deployment and custom task types, distributed
-scheduling, with overall scheduling capability increased linearly with the
-scale of the cluster
- - Support various task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Sub_Process, Procedure, etc.
- - Support scheduling of workflows and dependencies, manual scheduling to pause/stop/recover task, support failure task retry/alarm, recover specified nodes from failure, kill task, etc.
- - Associate the tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of the task in real-time.
- - WYSIWYG online editing tasks
- - Support the priority of workflows & tasks, task failover, and task timeout alarm or failure.
- - Support workflow global parameters and node customized parameter settings.
- - Support online upload/download/management of resource files, etc. Support online file creation and editing.
- - Support task log online viewing and scrolling and downloading, etc.
- - Support the viewing of Master/Worker CPU load, memory, and CPU usage metrics.
- - Support displaying workflow history in tree/Gantt chart, as well as statistical analysis on the task status & process status in each workflow.
- - Support back-filling data.
- - Support internationalization.
- - More features waiting for partners to explore...
 
-## What's in DolphinScheduler
+- Highly Reliable,
+DolphinScheduler uses a decentralized multi-master and multi-worker architecture, which naturally supports horizontal scaling and high availability
+- High performance, its performance is N times faster than other orchestration platform and it can support tens of millions of tasks per day
+- Supports multi-tenancy
+- Supports various task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, Hive, Spark SQL), Python, Procedure, Sub_Workflow,
+Http, K8s, Jupyter, MLflow, SageMaker, DVC, Pytorch, Amazon EMR, etc
+- Orchestrating workflows and dependencies, you can pause/stop/recover task any time, failed tasks can be set to automatically retry
+- Visualizing the running state of the task in real-time and seeing the task runtime log
+- What you see is what you get when you edit the task on the UI
+- Backfill can be operated on the UI directly
+- Perfect project, resource, data source-level permission control
+- Displaying workflow history in tree/Gantt chart, as well as statistical analysis on the task status & process status in each workflow
+- Supports internationalization
+- Cloud Native, DolphinScheduler supports orchestrating multi-cloud/data center workflow, and 
+supports custom task type
+- More features waiting for partners to explore
 
- Stability | Accessibility | Features | Scalability |
- --------- | ------------- | -------- | ------------|
-Decentralized multi-master and multi-worker | Visualization of workflow key information, such as task status, task type, retry times, task operation machine information, visual variables, and so on at a glance.  |  Support pause, recover operation | Support customized task types
-support HA | Visualization of all workflow operations, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, provide API mode operations. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs.  | The scheduler supports distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic adjustment.
-Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process |  |
 
 ## User Interface Screenshots
-
+![dag](./images/en_US/dag.png)
-![home page](https://user-images.githubusercontent.com/15833811/75218288-bf286400-57d4-11ea-8263-d639c6511d5f.jpg)
+<img width="1100" src="https://user-images.githubusercontent.com/15833811/197348110-1653ea32-ce07-436c-a0b8-6ac1af80aea5.png">
-![dag](https://user-images.githubusercontent.com/15833811/75236750-3374fe80-57f9-11ea-857d-62a66a5a559d.png)
+![data-source](./images/en_US/data-source.png)
-![process definition list page](https://user-images.githubusercontent.com/15833811/75216886-6f479e00-57d0-11ea-92dd-66e7640a186f.png)
+![home](./images/en_US/home.png)
-![view task log online](https://user-images.githubusercontent.com/15833811/75216924-9900c500-57d0-11ea-91dc-3522a76bdbbe.png)
+![master](./images/en_US/master.png)
-![resource management](https://user-images.githubusercontent.com/15833811/75216984-be8dce80-57d0-11ea-840d-58546edc8788.png)
+![workflow-tree](./images/en_US/workflow-tree.png)
-![monitor](https://user-images.githubusercontent.com/59273635/75625839-c698a480-5bfc-11ea-8bbe-895b561b337f.png)
-![security](https://user-images.githubusercontent.com/15833811/75236441-bfd2f180-57f8-11ea-88bd-f24311e01b7e.png)
-![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png)
 
 ## QuickStart in Docker
 
-Please refer the official website document: [QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/docker.html)
+Please refer the official website document: [QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/3.0.3/guide/start/docker)
 
 ## QuickStart in Kubernetes
 
-Please refer to the official website document: [QuickStart in Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/kubernetes.html)
+Please refer to the official website document: [QuickStart in Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/3.0.3/guide/installation/kubernetes)
 
 ## How to Build
 
@@ -84,29 +74,27 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-b
 dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-src.tar.gz: Source code package of DolphinScheduler
 ```
 
-## Thanks
-
-DolphinScheduler is based on a lot of excellent open-source projects, such as Google guava, guice, grpc, netty, quartz, and many open-source projects of Apache and so on.
-We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community!
-
 ## Get Help
 
 1. Submit an [issue](https://github.com/apache/dolphinscheduler/issues/new/choose)
-2. [Join our slack](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) and send your question to channel `#troubleshooting`
+2. [Join our slack](https://s.apache.org/dolphinscheduler-slack) and send your question to channel `#general`
+3. Send email to users@dolphinscheduler.apache.org or dev@dolphinscheduler.apache.org
 
 ## Community
 
 You are very welcome to communicate with the developers and users of Dolphin Scheduler. There are two ways to find them:
-1. Join the Slack channel [Slack](https://asf-dolphinscheduler.slack.com/).
+1. Join the Slack channel [Slack](https://asf-dolphinscheduler.slack.com/)
-2. Follow the [Twitter account of DolphinScheduler](https://twitter.com/dolphinschedule) and get the latest news on time.
+2. Follow the [Twitter account of DolphinScheduler](https://twitter.com/dolphinschedule) and get the latest news on time
-
-### Contributor over time
-
-[![Contributor over time](https://contributor-graph-api.apiseven.com/contributors-svg?chart=contributorOverTime&repo=apache/dolphinscheduler)](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/dolphinscheduler)
 
 ## How to Contribute
 
-The community welcomes everyone to contribute, please refer to this page to find out more: [How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html).
+The community welcomes everyone to contribute, please refer to this page to find out more: [How to contribute](docs/docs/en/contribute/join/contribute.md).
+
+
+## Thanks
+
+DolphinScheduler is based on a lot of excellent open-source projects, such as Google guava, grpc, netty, quartz, and many open-source projects of Apache and so on.
+We would like to express our deep gratitude to all the open-source projects used in DolphinScheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community
 
 # Landscapes
 
@@ -120,4 +108,4 @@ DolphinScheduler enriches the <a href="https://landscape.cncf.io/?landscape=obse
 
 ## License
 
-Please refer to the [LICENSE](https://github.com/apache/dolphinscheduler/blob/dev/LICENSE) file.
+Please refer to the [LICENSE](https://github.com/apache/dolphinscheduler/blob/dev/LICENSE) file

README_zh_CN.md

@@ -36,19 +36,11 @@ Dolphin Scheduler Official Website
 
 ## 系统部分截图
 
-![home page](https://user-images.githubusercontent.com/15833811/75208819-abbad000-57b7-11ea-8d3c-67e7c270671f.jpg)
+![dag](./images/zh_CN/dag.png)
-
+![data-source](./images/zh_CN/data-source.png)
-![dag](https://user-images.githubusercontent.com/15833811/75209584-93e44b80-57b9-11ea-952e-537fb24ec72d.jpg)
+![home](./images/zh_CN/home.png)
-
+![master](./images/zh_CN/master.png)
-![log](https://user-images.githubusercontent.com/15833811/75209645-c55d1700-57b9-11ea-94d4-e3fa91ab5218.jpg)
+![workflow-tree](./images/zh_CN/workflow-tree.png)
-
-![gantt](https://user-images.githubusercontent.com/15833811/75209640-c0986300-57b9-11ea-878e-a2098533ad44.jpg)
-
-![resources](https://user-images.githubusercontent.com/15833811/75209403-11f42280-57b9-11ea-9b59-d4be77063553.jpg)
-
-![monitor](https://user-images.githubusercontent.com/15833811/75209631-b5ddce00-57b9-11ea-8d22-cdf15cf0ee25.jpg)
-
-![security](https://user-images.githubusercontent.com/15833811/75209633-baa28200-57b9-11ea-9def-94bef2e212a7.jpg)
 
 ## 近期研发计划
 
@@ -57,15 +49,15 @@ DolphinScheduler的工作计划：<a href="https://github.com/apache/dolphinsche
 ## 参与贡献
 
 非常欢迎大家来参与贡献，贡献流程请参考：
-[[参与贡献](https://dolphinscheduler.apache.org/zh-cn/community/development/contribute.html)]
+[[参与贡献](docs/docs/zh/contribute/join/contribute.md)]
 
 ## 快速试用 Docker
 
-请参考官方文档: [快速试用 Docker 部署](http://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/guide/installation/docker.html)
+请参考官方文档: [快速试用 Docker 部署](https://dolphinscheduler.apache.org/zh-cn/docs/3.0.3/guide/start/docker)
 
 ## 快速试用 Kubernetes
 
-请参考官方文档: [快速试用 Kubernetes 部署](http://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/guide/installation/kubernetes.html)
+请参考官方文档: [快速试用 Kubernetes 部署](https://dolphinscheduler.apache.org/zh-cn/docs/3.0.3/guide/installation/kubernetes)
 
 ## 如何构建
 
@@ -88,11 +80,11 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g
 ## 获得帮助
 
 1. 提交 [issue](https://github.com/apache/dolphinscheduler/issues/new/choose)
-2. [加入slack群](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) 并在频道 `#troubleshooting` 中提问
+2. [加入slack群](https://s.apache.org/dolphinscheduler-slack) 并在频道 `#troubleshooting` 中提问
 
 ## 社区
 
-1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw)加入slack channel
+1. 通过[该申请链接](https://s.apache.org/dolphinscheduler-slack)加入slack channel
 2. 关注[Apache Dolphin Scheduler的Twitter账号](https://twitter.com/dolphinschedule)获取实时动态
 
 ## 版权

deploy/README.md

@@ -1,4 +1,4 @@
 # DolphinScheduler for Docker and Kubernetes
 
-* [Start Up DolphinScheduler with Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/docker.html)
+- [Start Up DolphinScheduler with Docker](https://dolphinscheduler.apache.org/en-us/docs/3.0.3/guide/start/docker)
-* [Start Up DolphinScheduler with Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/kubernetes.html)
+- [Start Up DolphinScheduler with Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/3.0.3/guide/installation/kubernetes)

deploy/docker/.env

@@ -15,9 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-HUB=ghcr.io/apache/dolphinscheduler
+HUB=apache
-TAG=latest
+TAG=3.0.5
 
 TZ=Asia/Shanghai
+DATABASE=postgresql
 SPRING_DATASOURCE_URL=jdbc:postgresql://dolphinscheduler-postgresql:5432/dolphinscheduler
 REGISTRY_ZOOKEEPER_CONNECT_STRING=dolphinscheduler-zookeeper:2181

deploy/docker/docker-compose.yml

@@ -19,6 +19,9 @@ version: "3.8"
 services:
   dolphinscheduler-postgresql:
     image: bitnami/postgresql:11.11.0
+    ports:
+      - "5432:5432"
+    profiles: ["all", "schema"]
     environment:
       POSTGRESQL_USERNAME: root
       POSTGRESQL_PASSWORD: root
@@ -35,6 +38,7 @@ services:
 
   dolphinscheduler-zookeeper:
     image: bitnami/zookeeper:3.6.2
+    profiles: ["all"]
     environment:
       ALLOW_ANONYMOUS_LOGIN: "yes"
       ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
@@ -51,7 +55,8 @@ services:
   dolphinscheduler-schema-initializer:
     image: ${HUB}/dolphinscheduler-tools:${TAG}
     env_file: .env
-    command: [ bin/create-schema.sh ]
+    profiles: ["schema"]
+    command: [ tools/bin/upgrade-schema.sh ]
     depends_on:
       dolphinscheduler-postgresql:
         condition: service_healthy
@@ -66,15 +71,15 @@ services:
     image: ${HUB}/dolphinscheduler-api:${TAG}
     ports:
       - "12345:12345"
+      - "25333:25333"
+    profiles: ["all"]
     env_file: .env
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+      test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
       interval: 30s
       timeout: 5s
       retries: 3
     depends_on:
-      dolphinscheduler-schema-initializer:
-        condition: service_completed_successfully
       dolphinscheduler-zookeeper:
         condition: service_healthy
     volumes:
@@ -86,15 +91,13 @@ services:
 
   dolphinscheduler-alert:
     image: ${HUB}/dolphinscheduler-alert-server:${TAG}
+    profiles: ["all"]
     env_file: .env
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
       interval: 30s
       timeout: 5s
       retries: 3
-    depends_on:
-      dolphinscheduler-schema-initializer:
-        condition: service_completed_successfully
     volumes:
       - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     networks:
@@ -102,6 +105,7 @@ services:
 
   dolphinscheduler-master:
     image: ${HUB}/dolphinscheduler-master:${TAG}
+    profiles: ["all"]
     env_file: .env
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
@@ -109,8 +113,6 @@ services:
       timeout: 5s
       retries: 3
     depends_on:
-      dolphinscheduler-schema-initializer:
-        condition: service_completed_successfully
       dolphinscheduler-zookeeper:
         condition: service_healthy
     volumes:
@@ -121,6 +123,7 @@ services:
 
   dolphinscheduler-worker:
     image: ${HUB}/dolphinscheduler-worker:${TAG}
+    profiles: ["all"]
     env_file: .env
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
@@ -128,8 +131,6 @@ services:
       timeout: 5s
       retries: 3
     depends_on:
-      dolphinscheduler-schema-initializer:
-        condition: service_completed_successfully
       dolphinscheduler-zookeeper:
         condition: service_healthy
     volumes:

deploy/docker/docker-stack.yml

@@ -50,6 +50,7 @@ services:
     image: apache/dolphinscheduler-api
     ports:
       - 12345:12345
+      - 25333:25333
     env_file: .env
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]

deploy/kubernetes/dolphinscheduler/Chart.yaml

@@ -35,18 +35,24 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 2.0.0
+version: 3.0.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: dev-SNAPSHOT
+appVersion: 3.0.5
 
 dependencies:
 - name: postgresql
   version: 10.3.18
-  repository: https://charts.bitnami.com/bitnami
+  # Due to a change in the Bitnami repo, https://charts.bitnami.com/bitnami was truncated only
+  # containing entries for the latest 6 months (from January 2022 on).
+  # This URL: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+  # contains the full 'index.yaml'.
+  # See detail here: https://github.com/bitnami/charts/issues/10833
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   condition: postgresql.enabled
 - name: zookeeper
   version: 6.5.3
-  repository: https://charts.bitnami.com/bitnami
+  # Same as above.
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   condition: zookeeper.enabled

deploy/kubernetes/dolphinscheduler/resources/config/common.properties

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if index .Values.conf "common" }}
+{{- range $key, $value := index .Values.conf "common" }}
+{{ $key }}={{ $value }}
+{{- end }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -112,7 +112,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 Create a default fully qualified zookkeeper quorum.
 */}}
 {{- define "dolphinscheduler.zookeeper.quorum" -}}
-{{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}}
+{{- $port := default "2181" .Values.zookeeper.service.port | toString -}}
 {{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port -}}
 {{- end -}}
 

deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-configs
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+data:
+  common_properties: |-
+{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -39,6 +39,7 @@ spec:
         {{- toYaml .Values.alert.annotations | nindent 8 }}
       {{- end }}
     spec:
+      serviceAccountName: {{ template "dolphinscheduler.fullname" . }}
       {{- if .Values.alert.affinity }}
       affinity:
         {{- toYaml .Values.alert.affinity | nindent 8 }}
@@ -67,6 +68,8 @@ spec:
           env:
             - name: TZ
               value: {{ .Values.timezone }}
+            - name: SPRING_JACKSON_TIME_ZONE
+              value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{ range $key, $value :=  .Values.alert.env }}
             - name: {{ $key }}
@@ -102,6 +105,9 @@ spec:
           volumeMounts:
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: config-volume
+              mountPath: /opt/dolphinscheduler/conf/common.properties
+              subPath: common_properties
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-alert
           {{- if .Values.alert.persistentVolumeClaim.enabled }}
@@ -110,3 +116,6 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
+        - name: config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-configs

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -39,6 +39,7 @@ spec:
         {{- toYaml .Values.api.annotations | nindent 8 }}
       {{- end }}
     spec:
+      serviceAccountName: {{ template "dolphinscheduler.fullname" . }}
       {{- if .Values.api.affinity }}
       affinity:
         {{- toYaml .Values.api.affinity | nindent 8 }}
@@ -62,9 +63,13 @@ spec:
           ports:
             - containerPort: 12345
               name: "api-port"
+            - containerPort: 25333
+              name: "python-api-port"
           env:
             - name: TZ
               value: {{ .Values.timezone }}
+            - name: SPRING_JACKSON_TIME_ZONE
+              value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
@@ -82,7 +87,7 @@ spec:
           {{- if .Values.api.livenessProbe.enabled }}
           livenessProbe:
             exec:
-              command: ["curl", "-s", "http://localhost:12345/actuator/health/liveness"]
+              command: ["curl", "-s", "http://localhost:12345/dolphinscheduler/actuator/health/liveness"]
             initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }}
@@ -92,7 +97,7 @@ spec:
           {{- if .Values.api.readinessProbe.enabled }}
           readinessProbe:
             exec:
-              command: ["curl", "-s", "http://localhost:12345/actuator/health/readiness"]
+              command: ["curl", "-s", "http://localhost:12345/dolphinscheduler/actuator/health/readiness"]
             initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }}
@@ -102,6 +107,9 @@ spec:
           volumeMounts:
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-api
+            - name: config-volume
+              mountPath: /opt/dolphinscheduler/conf/common.properties
+              subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
       volumes:
@@ -112,5 +120,8 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
+        - name: config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-configs
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}

deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml

@@ -38,10 +38,12 @@ spec:
           image: {{ include "dolphinscheduler.image.fullname.tools" . }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
-            - bin/create-schema.sh
+            - tools/bin/upgrade-schema.sh
           env:
             - name: TZ
               value: {{ .Values.timezone }}
+            - name: SPRING_JACKSON_TIME_ZONE
+              value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}

deploy/kubernetes/dolphinscheduler/templates/rbac.yaml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "dolphinscheduler.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+  name: {{ template "dolphinscheduler.fullname" . }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "dolphinscheduler.fullname" . }}
+  labels:
+    app: {{ template "dolphinscheduler.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "dolphinscheduler.fullname" . }}
+  labels:
+    app: {{ template "dolphinscheduler.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "dolphinscheduler.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "dolphinscheduler.fullname" . }}
+    namespace: {{ .Release.Namespace }}

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -36,6 +36,7 @@ spec:
         {{- toYaml .Values.master.annotations | nindent 8 }}
       {{- end }}
     spec:
+      serviceAccountName: {{ template "dolphinscheduler.fullname" . }}
       {{- if .Values.master.affinity }}
       affinity:
         {{- toYaml .Values.master.affinity | nindent 8 }}
@@ -62,6 +63,8 @@ spec:
           env:
             - name: TZ
               value: {{ .Values.timezone }}
+            - name: SPRING_JACKSON_TIME_ZONE
+              value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
@@ -100,6 +103,9 @@ spec:
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-master
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
+            - name: config-volume
+              mountPath: /opt/dolphinscheduler/conf/common.properties
+              subPath: common_properties
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-master
           {{- if .Values.master.persistentVolumeClaim.enabled }}
@@ -109,6 +115,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
+        - name: config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-configs
   {{- if .Values.master.persistentVolumeClaim.enabled }}
   volumeClaimTemplates:
     - metadata:

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -36,6 +36,7 @@ spec:
         {{- toYaml .Values.worker.annotations | nindent 8 }}
       {{- end }}
     spec:
+      serviceAccountName: {{ template "dolphinscheduler.fullname" . }}
       {{- if .Values.worker.affinity }}
       affinity:
         {{- toYaml .Values.worker.affinity | nindent 8 }}
@@ -62,7 +63,9 @@ spec:
           env:
             - name: TZ
               value: {{ .Values.timezone }}
-            - name: ALERT_LISTEN_HOST
+            - name: SPRING_JACKSON_TIME_ZONE
+              value: {{ .Values.timezone }}
+            - name: WORKER_ALERT_LISTEN_HOST
               value: {{ include "dolphinscheduler.fullname" . }}-alert
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
@@ -103,9 +106,13 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-worker-data
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
+            - name: config-volume
+              mountPath: /opt/dolphinscheduler/conf/common.properties
+              subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
       volumes:
+        {{- if .Values.worker.persistentVolumeClaim.enabled }}
         - name: {{ include "dolphinscheduler.fullname" . }}-worker-data
           {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
           persistentVolumeClaim:
@@ -120,6 +127,15 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
+        {{- else }}
+        - name: {{ include "dolphinscheduler.fullname" . }}-worker-data
+          emptyDir: {}
+        - name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
+          emptyDir: {}
+        {{- end }}
+        - name: config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-configs
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
   {{- if .Values.worker.persistentVolumeClaim.enabled }}

deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml

@@ -40,6 +40,13 @@ spec:
       {{- end }}
       protocol: TCP
       name: api-port
+    - port: 25333
+      targetPort: python-api-port
+      {{- if and (eq .Values.api.service.type "NodePort") .Values.api.service.nodePort }}
+      nodePort: {{ .Values.api.service.nodePort }}
+      {{- end }}
+      protocol: TCP
+      name: python-api-port
   {{- if .Values.api.service.externalIPs }}
   externalIPs:
     {{- toYaml .Values.api.service.externalIPs | nindent 4 }}

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -23,7 +23,7 @@ timezone: "Asia/Shanghai"
 
 image:
   registry: "dolphinscheduler.docker.scarf.sh/apache"
-  tag: "dev-SNAPSHOT"
+  tag: "3.0.5"
   pullPolicy: "IfNotPresent"
   pullSecret: ""
 
@@ -52,6 +52,8 @@ externalDatabase:
 ## If not exists external registry, the zookeeper registry will be used by default.
 zookeeper:
   enabled: true
+  service:
+    port: 2181 
   fourlwCommandsWhitelist: "srvr,ruok,wchs,cons"
   persistence:
     enabled: false
@@ -64,6 +66,84 @@ externalRegistry:
   registryPluginName: "zookeeper"
   registryServers: "127.0.0.1:2181"
 
+conf:
+  common:
+    # user data local directory path, please make sure the directory exists and have read write permissions
+    data.basedir.path: /tmp/dolphinscheduler
+
+    # resource storage type: HDFS, S3, NONE
+    resource.storage.type: NONE
+
+    # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
+    resource.upload.path: /dolphinscheduler
+
+    # whether to startup kerberos
+    hadoop.security.authentication.startup.state: false
+
+    # java.security.krb5.conf path
+    java.security.krb5.conf.path: /opt/krb5.conf
+
+    # login user from keytab username
+    login.user.keytab.username: hdfs-mycluster@ESZ.COM
+
+    # login user from keytab path
+    login.user.keytab.path: /opt/hdfs.headless.keytab
+
+    # kerberos expire time, the unit is hour
+    kerberos.expire.time: 2
+    # resource view suffixs
+    #resource.view.suffixs: txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
+    # if resource.storage.type: HDFS, the user must have the permission to create directories under the HDFS root path
+    hdfs.root.user: hdfs
+    # if resource.storage.type: S3, the value like: s3a://dolphinscheduler; if resource.storage.type: HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
+    fs.defaultFS: file:///
+    aws.access.key.id: minioadmin
+    aws.secret.access.key: minioadmin
+    aws.region: us-east-1
+    aws.endpoint: http://localhost:9000
+    # resourcemanager port, the default value is 8088 if not specified
+    resource.manager.httpaddress.port: 8088
+    # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
+    yarn.resourcemanager.ha.rm.ids: 192.168.xx.xx,192.168.xx.xx
+    # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
+    yarn.application.status.address: http://ds1:%s/ws/v1/cluster/apps/%s
+    # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
+    yarn.job.history.status.address: http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
+
+    # datasource encryption enable
+    datasource.encryption.enable: false
+
+    # datasource encryption salt
+    datasource.encryption.salt: '!@#$%^&*'
+
+    # data quality option
+    data-quality.jar.name: dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+
+    #data-quality.error.output.path: /tmp/data-quality-error-data
+
+    # Network IP gets priority, default inner outer
+
+    # Whether hive SQL is executed in the same session
+    support.hive.oneSession: false
+
+    # use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
+    sudo.enable: true
+
+    # network interface preferred like eth0, default: empty
+    #dolphin.scheduler.network.interface.preferred:
+
+    # network IP gets priority, default: inner outer
+    #dolphin.scheduler.network.priority.strategy: default
+
+    # system env path
+    #dolphinscheduler.env.path: dolphinscheduler_env.sh
+    # development state
+    development.state: false
+    # rpc port
+    alert.rpc.port: 50052
+    # Url endpoint for zeppelin RESTful API
+    zeppelin.rest.url: http://localhost:8080
+
 common:
   ## Configmap
   configmap:
@@ -178,12 +258,14 @@ master:
     MASTER_EXEC_TASK_NUM: "20"
     MASTER_DISPATCH_TASK_NUM: "3"
     MASTER_HOST_SELECTOR: "LowerWeight"
-    MASTER_HEARTBEAT_INTERVAL: "10"
+    MASTER_HEARTBEAT_INTERVAL: "10s"
+    MASTER_HEARTBEAT_ERROR_THRESHOLD: "5"
     MASTER_TASK_COMMIT_RETRYTIMES: "5"
-    MASTER_TASK_COMMIT_INTERVAL: "1000"
+    MASTER_TASK_COMMIT_INTERVAL: "1s"
+    MASTER_STATE_WHEEL_INTERVAL: "5s"
     MASTER_MAX_CPU_LOAD_AVG: "-1"
     MASTER_RESERVED_MEMORY: "0.3"
-    MASTER_FAILOVER_INTERVAL: 10
+    MASTER_FAILOVER_INTERVAL: "10m"
     MASTER_KILL_YARN_JOB_WHEN_HANDLE_FAILOVER: "true"
 
 worker:
@@ -257,7 +339,8 @@ worker:
     WORKER_MAX_CPU_LOAD_AVG: "-1"
     WORKER_RESERVED_MEMORY: "0.3"
     WORKER_EXEC_THREADS: "100"
-    WORKER_HEARTBEAT_INTERVAL: "10"
+    WORKER_HEARTBEAT_INTERVAL: "10s"
+    WORKER_HEART_ERROR_THRESHOLD: "5"
     WORKER_HOST_WEIGHT: "100"
     WORKER_GROUPS: "default"
 

docs/configs/index.md.jsx

@@ -43,6 +43,7 @@ import docs202Config from '../../../site_config/docs2-0-2';
 import docs203Config from '../../../site_config/docs2-0-3';
 import docs205Config from '../../../site_config/docs2-0-5';
 import docs300Config from '../../../site_config/docs3-0-0';
+import docs301Config from '../../../site_config/docs3-0-1';
 import docsDevConfig from '../../../site_config/docsdev';
 
 const docsSource = {
@@ -62,6 +63,7 @@ const docsSource = {
   '2.0.3': docs203Config,
   '2.0.5': docs205Config,
   '3.0.0': docs300Config,
+  '3.0.1': docs301Config,
   dev: docsDevConfig,
 };
 

docs/configs/site.js

@@ -24,13 +24,13 @@ export default {
   port: 8080,
   domain: 'dolphinscheduler.apache.org',
   copyToDist: ['asset', 'img', 'file', '.asf.yaml', 'sitemap.xml', '.nojekyll', '.htaccess', 'googled0df7b96f277a143.html'],
-  docsLatest: '3.0.0',
+  docsLatest: '3.0.1',
   defaultSearch: 'google', // default search engine
   defaultLanguage: 'en-us',
   'en-us': {
     banner: {
       text: '🤔 Have queries regarding Apache DolphinScheduler, Join Slack channel to disscuss them ',
-      link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw'
+      link: 'https://s.apache.org/dolphinscheduler-slack'
     },
     pageMenu: [
       {
@@ -45,29 +45,19 @@ export default {
         children: [
           {
             key: 'docs0',
-            text: 'latest(3.0.0-alpha)',
+            text: 'latest(3.0.1)',
             link: '/en-us/docs/latest/user_doc/about/introduction.html',
           },
           {
             key: 'docs1',
-            text: '1.3.9',
+            text: '2.0.6',
-            link: '/en-us/docs/1.3.9/user_doc/quick-start.html',
+            link: '/en-us/docs/2.0.6/user_doc/guide/quick-start.html',
           },
           {
             key: 'docsHistory',
             text: 'Older Versions',
             link: '/en-us/docs/release/history-versions.html',
-          },
+          }
-          {
-            key: 'pythonAPI',
-            text: 'PyDolphinScheduler',
-            link: '/python/index.html',
-          },
-          {
-            key: 'docsdev',
-            text: 'dev',
-            link: '/en-us/docs/dev/user_doc/about/introduction.html',
-          },
         ],
       },
       {
@@ -79,15 +69,10 @@ export default {
         text: 'BLOG',
         link: '/en-us/blog/index.html',
       },
-      {
-        key: 'development',
-        text: 'DEVELOPMENT',
-        link: '/en-us/development/development-environment-setup.html',
-      },
       {
         key: 'community',
         text: 'COMMUNITY',
-        link: '/en-us/community/team.html',
+        link: '/en-us/community/community.html',
       },
       {
         key: 'ASF',
@@ -139,57 +124,21 @@ export default {
         link: '/en-us/user/index.html',
       },
     ],
-    documentation: {
-      title: 'Documentation',
-      list: [
-        {
-          text: 'Overview',
-          link: '/en-us/development/architecture-design.html',
-        },
-        {
-          text: 'Quick start',
-          link: '/en-us/docs/latest/user_doc/guide/quick-start.html',
-        },
-        {
-          text: 'Developer guide',
-          link: '/en-us/development/development-environment-setup.html',
-        },
-      ],
-    },
-    asf: {
-      title: 'ASF',
-      list: [
-        {
-          text: 'Foundation',
-          link: 'http://www.apache.org',
-        },
-        {
-          text: 'License',
-          link: 'http://www.apache.org/licenses/',
-        },
-        {
-          text: 'Events',
-          link: 'http://www.apache.org/events/current-event',
-        },
-        {
-          text: 'Sponsorship',
-          link: 'http://www.apache.org/foundation/sponsorship.html',
-        },
-        {
-          text: 'Thanks',
-          link: 'http://www.apache.org/foundation/thanks.html',
-        },
-      ],
-    },
     contact: {
       title: 'About us',
       content: 'Do you need feedback? Please contact us through the following ways.',
       list: [
+        {
+          name: 'Slack',
+          img1: '/img/slack.png',
+          img2: '/img/slack-selected.png',
+          link: 'https://s.apache.org/dolphinscheduler-slack',
+        },
         {
           name: 'Email List',
           img1: '/img/emailgray.png',
           img2: '/img/emailblue.png',
-          link: '/en-us/community/development/subscribe.html',
+          link: '/en-us/docs/latest/user_doc/contribute/join/subscribe.html',
         },
         {
           name: 'Twitter',
@@ -197,26 +146,14 @@ export default {
           img2: '/img/twitterblue.png',
           link: 'https://twitter.com/dolphinschedule',
         },
-        {
-          name: 'Stack Overflow',
-          img1: '/img/stackoverflow.png',
-          img2: '/img/stackoverflow-selected.png',
-          link: 'https://stackoverflow.com/questions/tagged/apache-dolphinscheduler',
-        },
-        {
-          name: 'Slack',
-          img1: '/img/slack.png',
-          img2: '/img/slack-selected.png',
-          link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw',
-        },
       ],
     },
-    copyright: 'Copyright © 2019-2021 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
+    copyright: 'Copyright © 2019-2022 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
   },
   'zh-cn': {
     banner: {
       text: '🤔 有关于 Apache DolphinScheduler 的疑问，加入 Slack 频道来讨论他们 ',
-      link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw'
+      link: 'https://s.apache.org/dolphinscheduler-slack'
     },
     pageMenu: [
       {
@@ -231,29 +168,19 @@ export default {
         children: [
           {
             key: 'docs0',
-            text: '最新版本latest(3.0.0-alpha)',
+            text: '最新版本latest(3.0.1)',
             link: '/zh-cn/docs/latest/user_doc/about/introduction.html',
           },
           {
             key: 'docs1',
-            text: '1.3.9',
+            text: '2.0.6',
-            link: '/zh-cn/docs/1.3.9/user_doc/quick-start.html',
+            link: '/zh-cn/docs/2.0.6/user_doc/guide/quick-start.html',
           },
           {
             key: 'docsHistory',
             text: '历史版本',
             link: '/zh-cn/docs/release/history-versions.html',
-          },
+          }
-          {
-            key: 'pythonAPI',
-            text: 'PyDolphinScheduler',
-            link: '/python/index.html',
-          },
-          {
-            key: 'docsdev',
-            text: 'dev',
-            link: '/zh-cn/docs/dev/user_doc/about/introduction.html',
-          },
         ],
       },
       {
@@ -266,15 +193,10 @@ export default {
         text: '博客',
         link: '/zh-cn/blog/index.html',
       },
-      {
-        key: 'development',
-        text: '开发者',
-        link: '/zh-cn/development/development-environment-setup.html',
-      },
       {
         key: 'community',
         text: '社区',
-        link: '/zh-cn/community/team.html',
+        link: '/zh-cn/community/community.html',
       },
       {
         key: 'ASF',
@@ -327,57 +249,21 @@ export default {
         link: '/zh-cn/user/index.html',
       },
     ],
-    documentation: {
-      title: '文档',
-      list: [
-        {
-          text: '概览',
-          link: '/zh-cn/development/architecture-design.html',
-        },
-        {
-          text: '快速开始',
-          link: '/zh-cn/docs/latest/user_doc/guide/quick-start.html',
-        },
-        {
-          text: '开发者指南',
-          link: '/zh-cn/development/development-environment-setup.html',
-        },
-      ],
-    },
-    asf: {
-      title: 'ASF',
-      list: [
-        {
-          text: '基金会',
-          link: 'http://www.apache.org',
-        },
-        {
-          text: '证书',
-          link: 'http://www.apache.org/licenses/',
-        },
-        {
-          text: '事件',
-          link: 'http://www.apache.org/events/current-event',
-        },
-        {
-          text: '赞助',
-          link: 'http://www.apache.org/foundation/sponsorship.html',
-        },
-        {
-          text: '致谢',
-          link: 'http://www.apache.org/foundation/thanks.html',
-        },
-      ],
-    },
     contact: {
       title: '联系我们',
       content: '有问题需要反馈？请通过以下方式联系我们。',
       list: [
+        {
+          name: 'Slack',
+          img1: '/img/slack.png',
+          img2: '/img/slack-selected.png',
+          link: 'https://s.apache.org/dolphinscheduler-slack',
+        },
         {
           name: '邮件列表',
           img1: '/img/emailgray.png',
           img2: '/img/emailblue.png',
-          link: '/zh-cn/community/development/subscribe.html',
+          link: '/zh-cn/docs/latest/user_doc/contribute/join/subscribe.html',
         },
         {
           name: 'Twitter',
@@ -385,20 +271,8 @@ export default {
           img2: '/img/twitterblue.png',
           link: 'https://twitter.com/dolphinschedule',
         },
-        {
-          name: 'Stack Overflow',
-          img1: '/img/stackoverflow.png',
-          img2: '/img/stackoverflow-selected.png',
-          link: 'https://stackoverflow.com/questions/tagged/apache-dolphinscheduler',
-        },
-        {
-          name: 'Slack',
-          img1: '/img/slack.png',
-          img2: '/img/slack-selected.png',
-          link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw',
-        },
       ],
     },
-    copyright: 'Copyright © 2019-2021 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
+    copyright: 'Copyright © 2019-2022 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
   },
 };

docs/docs/en/DSIP.md

@@ -0,0 +1,91 @@
+# DSIP
+
+DolphinScheduler Improvement Proposal(DSIP) introduce major improvements to the Apache DolphinScheduler codebase. It is
+not for small incremental improvements, and the purpose of DSIP is to notice and inform community the finished or coming
+big feature for Apache DolphinScheduler.
+
+## What is considered as DSIP
+
+- Any major new feature, major improvement, introduce or remove components
+- Any major change of public interfaces, such as API endpoints, web ui huge change
+
+When the change in doubt and any committer thinks it should be DSIP, it does.
+
+We use GitHub Issue and Apache mail thread to record and hold DSIP, for more detail you could go to section
+[current DSIPs](#current-dsips) and [past DSIPs](#past-dsips).
+
+As a DSIP, it should:
+
+- Have a mail thread title started with `[DISCUSS]` in [dev@dolphinscheduler.apache.org][mail-to-dev]
+- Have a GitHub Issue labeled with `DSIP`, and including the mail thread link in the description.
+
+### Current DSIPs
+
+Current DSIPs including all DSIP still work-in-progress, you could see in [current DSIPs][current-DSIPs]
+
+### Past DSIPs
+
+Past DSIPs including all DSIP already done or retired for some reason, you could see in [past DSIPs][past-DSIPs]
+
+## DSIP Process
+
+### Create GitHub Issue
+
+All DSIP should start with GitHub Issue
+
+- If you pretty sure your issue is DSIP, you could click and choose "DSIP" in
+  [GitHub Issue][github-issue-choose]
+- If you not sure about your issue is DSIP or not, you could click and choose "Feature request" in
+  [GitHub Issue][github-issue-choose]. DolphinScheduler maintainer team would add label `DSIP`, mention you in the
+  issue and lead you to this document when they think it should be DSIP.
+
+You should add special prefix `[DSIP-XXX]`, `XXX` stand for the id DSIP. It's auto increment, and you could find the next
+integer in [All DSIPs][all-DSIPs] issues.
+
+### Send Discuss Mail
+
+After issue labeled with "DSIP", you should send an email to [dev@dolphinscheduler.apache.org][mail-to-dev].
+Describe the purpose, and the draft design about your idea.
+
+Here is the template for mail
+
+- Title: `[DISCUSS][DSIP-XXX] <CHANGE-TO-YOUR-LOVELY-PROPOSAL-TITLE>`, change `XXX` to special integer you just change in
+  [GitHub Issue](#create-github-issue), and also change proposal title.
+- Content:
+
+  ```text
+  Hi community,
+  
+  <CHANGE-TO-YOUR-PROPOSAL-DETAIL>
+  
+  I already add a GitHub Issue for my proposal, which you could see in <CHANGE-TO-YOUR-GITHUB-ISSUE-LINK>.
+  
+  Looking forward any feedback for this thread.
+  ```
+
+After community discuss and all of them think it worth as DSIP, you could [work on it](#work-on-it-or-create-subtask-for-it).
+But if community think it should not be DSIP or even this change should not be included to DolphinScheduler, maintainers
+terminate mail thread and remove label "DSIP" for GitHub Issue, or even close issue if it should not change.
+
+### Work On It, Or Create Subtask For It
+
+When your proposal pass in the mail thread, you could make your hand dirty and start the work. You could submit related
+pull requests in GitHub if change should in one single commit. What's more, if proposal is too huge in single commit, you
+could create subtasks in GitHub Issue like [DSIP-1][DSIP-1], and separate into multiple commit.
+
+### Close After It Done
+
+When DSIP is finished and all related PR were merged, you should reply the mail thread you created in
+[step two](#send-discuss-mail) to notice community the result of the DSIP. After that, this DSIP GitHub Issue would be
+closed and transfer from [current DSIPs][current-DSIPs] to [past DSIPs][past-DSIPs], but you could still find it in [All DSIPs][all-DSIPs]
+
+## An Example For DSIP
+
+* [[DSIP-1][Feature][Parent] Add Python API for DolphinScheduler][DSIP-1]: Have multiple subtasks and Projects on it.
+
+[all-DSIPs]: https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue+label%3A%22DSIP%22+
+[current-DSIPs]: https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue+is%3Aopen+label%3A%22DSIP%22
+[past-DSIPs]: https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue+is%3Aclosed+label%3A%22DSIP%22+
+[github-issue-choose]: https://github.com/apache/dolphinscheduler/issues/new/choose
+[mail-to-dev]: mailto:dev@dolphinscheduler.apache.org
+[DSIP-1]: https://github.com/apache/dolphinscheduler/issues/6407

docs/docs/en/about/features.md

@@ -0,0 +1,19 @@
+# Features
+
+## Simple to Use
+
+- **Visual DAG**: User-friendly drag-and-drop workflow definition and facility for run-time control.
+- **Modular Operation**: Modularity facilitates easy customization and maintenance.
+
+## Rich Scenarios
+
+- **Multiple Task Type Support**: Supports more than 10 task types, like Shell, MR, Spark, SQL, etc., with cross-language support making it easy to extend
+- **Workflow Ops**: Workflow can be timed, paused, resumed and stopped, enabling easy maintenance and control of global and local parameters.
+
+## High Reliability
+
+- **Reliability**: Decentralized designs ensure stability. Self-supporting HA task queue to avoid overload fault tolerant capability. DolphinScheduler facilitates a highly robust environment.
+
+## High Scalability
+
+- **Scalability**: Supports multitenancy and online resource management. Stable operation of 100,000 data tasks per day is supported.

docs/docs/en/about/glossary.md

@@ -9,12 +9,7 @@ scheduling system
 form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until
 there are no subsequent nodes. Examples are as follows:
 
-<p align="center">
+![about-glossary](../../../img/new_ui/dev/about/glossary.png)
-  <img src="/img/dag_examples_cn.jpg" alt="dag example"  width="60%" />
-  <p align="center">
-        <em>dag example</em>
-  </p>
-</p>
 
 **Process definition**: Visualization formed by dragging task nodes and establishing task node associations**DAG**
 

docs/docs/en/about/hardware.md

@@ -1,22 +1,24 @@
 # Hardware Environment
 
-DolphinScheduler, as an open-source distributed workflow task scheduling system, can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments and supports mainstream Linux operating system environments.
+This section briefs about the hardware requirements for DolphinScheduler. DolphinScheduler works as an open-source distributed workflow task scheduling system. It can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments. It also supports mainstream Linux operating system environments and ARM architecture.
 
 ## Linux Operating System Version Requirements
 
-| OS       | Version         |
+The Linux operating systems specified below can run on physical servers and mainstream virtualization environments such as VMware, KVM, and XEN.
+
+| Operating System | Version         |
 | :----------------------- | :----------: |
 | Red Hat Enterprise Linux | 7.0 and above   |
 | CentOS                   | 7.0 and above   |
 | Oracle Enterprise Linux  | 7.0 and above   |
 | Ubuntu LTS               | 16.04 and above |
 
-> **Attention:**
+> **Note:**
 >The above Linux operating systems can run on physical servers and mainstream virtualization environments such as VMware, KVM, and XEN.
 
-## Recommended Server Configuration
+## Server Configuration
 
-DolphinScheduler supports 64-bit hardware platforms with Intel x86-64 architecture. The following shows the recommended server requirements in a production environment:
+DolphinScheduler supports 64-bit hardware platforms with Intel x86-64 architecture. The following table shows the recommended server requirements in a production environment:
 
 ### Production Environment
 
@@ -24,11 +26,10 @@ DolphinScheduler supports 64-bit hardware platforms with Intel x86-64 architectu
 | --- | --- | --- | --- | --- |
 | 4 core+ | 8 GB+ | SAS | GbE | 1+ |
 
-> **Attention:**
+> **Note:**
 > - The above recommended configuration is the minimum configuration for deploying DolphinScheduler. Higher configuration is strongly recommended for production environments.
 > - The recommended hard disk size is more than 50GB and separate the system disk and data disk.
 
-
 ## Network Requirements
 
 DolphinScheduler provides the following network port configurations for normal operation:
@@ -39,10 +40,14 @@ DolphinScheduler provides the following network port configurations for normal o
 | WorkerServer | 1234  | not the communication port, require the native ports do not conflict |
 | ApiApplicationServer |  12345 | backend communication port |
 
-> **Attention:**
+> **Note:**
 > - MasterServer and WorkerServer do not need to enable communication between the networks. As long as the local ports do not conflict.
 > - Administrators can adjust relevant ports on the network side and host-side according to the deployment plan of DolphinScheduler components in the actual environment.
 
 ## Browser Requirements
 
-DolphinScheduler recommends Chrome and the latest browsers which use Chrome Kernel to access the front-end UI page.
+The minimum supported version of Google Chrome is version 85, but version 90 or above is recommended.
+
+## Synchronize clocks
+
+To avoid problems with internal cluster communications that can impact your task execution, make sure that the clocks on all of the cluster nodes are synchronized from a common clock source, such as using Chrony and/or NTP. Synchronizing the time ensures that every node in the cluster has the same time.

docs/docs/en/about/introduction.md

@@ -1,19 +1,7 @@
 # About DolphinScheduler
 
-Apache DolphinScheduler is a distributed, easy to extend visual DAG workflow task scheduling open-source system. Solves the intricate dependencies of data R&D ETL and the inability to monitor the health status of tasks. DolphinScheduler assembles tasks in the DAG streaming way, which can monitor the execution status of tasks in time, and supports operations like retry, recovery failure from specified nodes, pause, resume and kill tasks, etc.
+Apache DolphinScheduler provides a distributed and easy to expand visual workflow task scheduling open-source platform. It is suitable for enterprise-level scenarios. It provides a solution to visualize operation tasks, workflows, and the entire data processing procedures.
 
-## Simple to Use
+Apache DolphinScheduler aims to solve complex big data task dependencies and to trigger relationships in data OPS orchestration for various big data applications. Solves the intricate dependencies of data R&D ETL and the inability to monitor the health status of tasks. DolphinScheduler assembles tasks in the Directed Acyclic Graph (DAG) streaming mode, which can monitor the execution status of tasks in time, and supports operations like retry, recovery failure from specified nodes, pause, resume, and kill tasks, etc.
 
-- DolphinScheduler has DAG monitoring user interfaces, users can customize DAG by dragging and dropping. All process definitions are visualized, supports rich third-party systems APIs and one-click deployment.
+![Apache DolphinScheduler](../../../img/introduction_ui.png)
-
-## High Reliability
-
-- Decentralized multi-masters and multi-workers, support HA, select queues to avoid overload.
-
-## Rich Scenarios
-
-- Support features like multi-tenants, suspend and resume operations to cope with big data scenarios. Support many task types like Spark, Flink, Hive, MR, shell, python, sub_process.
-
-## High Scalability
-
-- Supports customized task types, distributed scheduling, and the overall scheduling capability increases linearly with the scale of the cluster.

docs/docs/en/architecture/configuration.md

@@ -4,80 +4,105 @@
 
 ## Preface
 
-This document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.
+This document explains the DolphinScheduler application configurations.
 
 ## Directory Structure
 
-Currently, all the configuration files are under [conf ] directory. 
+The directory structure of DolphinScheduler is as follows:
-Check the following simplified DolphinScheduler installation directories to have a direct view about the position of [conf] directory and configuration files it has. 
-This document only describes DolphinScheduler configurations and other topics are not going into.
-
-[Note: the DolphinScheduler (hereinafter called the ‘DS’) .]
 
 ```
-├─bin                               DS application commands directory
+├── LICENSE
-│  ├─dolphinscheduler-daemon.sh         startup or shutdown DS application 
+│
-│  ├─start-all.sh                       startup all DS services with configurations
+├── NOTICE
-│  ├─stop-all.sh                        shutdown all DS services with configurations
+│
-├─conf                              configurations directory
+├── licenses                                    directory of licenses
-│  ├─application-api.properties         API-service config properties
+│
-│  ├─datasource.properties              datasource config properties
+├── bin                                         directory of DolphinScheduler application commands, configrations scripts 
-│  ├─zookeeper.properties               ZooKeeper config properties
+│   ├── dolphinscheduler-daemon.sh              script to start or shut down DolphinScheduler application
-│  ├─master.properties                  master-service config properties
+│   ├── env                                     directory of scripts to load environment variables
-│  ├─worker.properties                  worker-service config properties
+│   │   ├── dolphinscheduler_env.sh             script to export environment variables [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] when you start or stop service using script `dolphinscheduler-daemon.sh`
-│  ├─quartz.properties                  quartz config properties
+│   │   └── install_env.sh                      script to export environment variables for DolphinScheduler installation when you use scripts `install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`
-│  ├─common.properties                  common-service [storage] config properties
+│   ├── install.sh                              script to auto-setup services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
-│  ├─alert.properties                   alert-service config properties
+│   ├── remove-zk-node.sh                       script to cleanup ZooKeeper caches 
-│  ├─config                             environment variables config directory
+│   ├── scp-hosts.sh                            script to copy installation files to target hosts 
-│      ├─install_config.conf                DS environment variables configuration script [install or start DS]
+│   ├── start-all.sh                            script to start all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
-│  ├─env                                load environment variables configs script directory
+│   ├── status-all.sh                           script to check the status of all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
-│      ├─dolphinscheduler_env.sh            load environment variables configs [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
+│   └── stop-all.sh                             script to shut down all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
-│  ├─org                                mybatis mapper files directory
+│
-│  ├─i18n                               i18n configs directory
+├── alert-server                                directory of DolphinScheduler alert-server commands, configrations scripts and libs
-│  ├─logback-api.xml                    API-service log config
+│   ├── bin
-│  ├─logback-master.xml                 master-service log config
+│   │   └── start.sh                            script to start DolphinScheduler alert-server
-│  ├─logback-worker.xml                 worker-service log config
+│   ├── conf
-│  ├─logback-alert.xml                  alert-service log config
+│   │   ├── application.yaml                    configurations of alert-server
-├─sql                                   .sql files to create or upgrade DS metadata
+│   │   ├── common.properties                   configurations of common-service like storage, credentials, etc. 
-│  ├─create                             create SQL scripts directory
+│   │   ├── dolphinscheduler_env.sh             script to load environment variables for alert-server
-│  ├─upgrade                            upgrade SQL scripts directory
+│   │   └── logback-spring.xml                  configurations of alert-service log
-│  ├─dolphinscheduler_postgre.sql       PostgreSQL database init script
+│   └── libs                                    directory of alert-server libs
-│  ├─dolphinscheduler_mysql.sql         MySQL database init script
+│
-│  ├─soft_version                       current DS version-id file
+├── api-server                                  directory of DolphinScheduler api-server commands, configrations scripts and libs
-├─script                            DS services deployment, database create or upgrade scripts directory
+│   ├── bin
-│  ├─create-dolphinscheduler.sh         DS database init script
+│   │   └── start.sh                            script to start DolphinScheduler api-server
-│  ├─upgrade-dolphinscheduler.sh        DS database upgrade script
+│   ├── conf
-│  ├─monitor-server.sh                  DS monitor-server start script       
+│   │   ├── application.yaml                    configurations of api-server
-│  ├─scp-hosts.sh                       transfer installation files script                                     
+│   │   ├── common.properties                   configurations of common-service like storage, credentials, etc.
-│  ├─remove-zk-node.sh                  cleanup ZooKeeper caches script       
+│   │   ├── dolphinscheduler_env.sh             script to load environment variables for api-server
-├─ui                                front-end web resources directory
+│   │   └── logback-spring.xml                  configurations of api-service log
-├─lib                               DS .jar dependencies directory
+│   ├── libs                                    directory of api-server libs
-├─install.sh                        auto-setup DS services script
+│   └── ui                                      directory of api-server related front-end web resources 
+│
+├── master-server                               directory of DolphinScheduler master-server commands, configrations scripts and libs
+│   ├── bin                                
+│   │   └── start.sh                            script to start DolphinScheduler master-server
+│   ├── conf
+│   │   ├── application.yaml                    configurations of master-server
+│   │   ├── common.properties                   configurations of common-service like storage, credentials, etc.
+│   │   ├── dolphinscheduler_env.sh             script to load environment variables for master-server
+│   │   └── logback-spring.xml                  configurations of master-service log
+│   └── libs                                    directory of master-server libs
+│
+├── standalone-server                           directory of DolphinScheduler standalone-server commands, configrations scripts and libs
+│   ├── bin
+│   │   └── start.sh                            script to start DolphinScheduler standalone-server
+│   ├── conf
+│   │   ├── application.yaml                    configurations of standalone-server
+│   │   ├── common.properties                   configurations of common-service like storage, credentials, etc.
+│   │   ├── dolphinscheduler_env.sh             script to load environment variables for standalone-server
+│   │   ├── logback-spring.xml                  configurations of standalone-service log
+│   │   └── sql                                 .sql files to create or upgrade DolphinScheduler metadata
+│   ├── libs                                    directory of standalone-server libs
+│   └── ui                                      directory of standalone-server related front-end web resources
+│       
+├── tools                                       directory of DolphinScheduler metadata tools commands, configrations scripts and libs
+│   ├── bin
+│   │   └── upgrade-schema.sh                   script to initialize or upgrade DolphinScheduler metadata
+│   ├── conf
+│   │   ├── application.yaml                    configurations of tools
+│   │   └── common.properties                   configurations of common-service like storage, credentials, etc.
+│   ├── libs                                    directory of tool libs
+│   └── sql                                     .sql files to create or upgrade DolphinScheduler metadata
+│     
+├── worker-server                               directory of DolphinScheduler worker-server commands, configrations scripts and libs
+│       ├── bin
+│       │   └── start.sh                        script to start DolphinScheduler worker-server
+│       ├── conf
+│       │   ├── application.yaml                configurations of worker-server
+│       │   ├── common.properties               configurations of common-service like storage, credentials, etc.
+│       │   ├── dolphinscheduler_env.sh         script to load environment variables for worker-server
+│       │   └── logback-spring.xml              configurations of worker-service log
+│       └── libs                                directory of worker-server libs
+│
+└── ui                                          directory of front-end web resources
 ```
 
 ## Configurations in Details
 
-serial number| service classification| config file|
-|--|--|--|
-1|startup or shutdown DS application|dolphinscheduler-daemon.sh
-2|datasource config properties|datasource.properties
-3|ZooKeeper config properties|zookeeper.properties
-4|common-service[storage] config properties|common.properties
-5|API-service config properties|application-api.properties
-6|master-service config properties|master.properties
-7|worker-service config properties|worker.properties
-8|alert-service config properties|alert.properties
-9|quartz config properties|quartz.properties
-10|DS environment variables configuration script[install/start DS]|install_config.conf
-11|load environment variables configs <br /> [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]|dolphinscheduler_env.sh
-12|services log config files|API-service log config : logback-api.xml  <br /> master-service log config  : logback-master.xml    <br /> worker-service log config : logback-worker.xml  <br /> alert-service log config : logback-alert.xml
 
 
-### dolphinscheduler-daemon.sh [startup or shutdown DS application]
+### dolphinscheduler-daemon.sh [startup or shutdown DolphinScheduler application]
 
-dolphinscheduler-daemon.sh is responsible for DS startup and shutdown.
+dolphinscheduler-daemon.sh is responsible for DolphinScheduler startup and shutdown.
 Essentially, start-all.sh or stop-all.sh startup and shutdown the cluster via dolphinscheduler-daemon.sh.
-Currently, DS just makes a basic config, remember to config further JVM options based on your practical situation of resources.
+Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical situation of resources.
 
 Default simplified parameters are:
 ```bash
@@ -94,306 +119,238 @@ export DOLPHINSCHEDULER_OPTS="
 "
 ```
 
-> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DS dependent on Netty to communicate).
+> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to communicate).
 
-### datasource.properties [datasource config properties]
+### Database connection related configuration
 
-DS uses Druid to manage database connections and default simplified configs are:
+DolphinScheduler uses Spring Hikari to manage database connections, configuration file location:
-|Parameters | Default value| Description|
-|--|--|--|
-spring.datasource.driver-class-name||datasource driver
-spring.datasource.url||datasource connection url
-spring.datasource.username||datasource username
-spring.datasource.password||datasource password
-spring.datasource.initialSize|5| initial connection pool size number
-spring.datasource.minIdle|5| minimum connection pool size number
-spring.datasource.maxActive|5| maximum connection pool size number
-spring.datasource.maxWait|60000| max wait milliseconds
-spring.datasource.timeBetweenEvictionRunsMillis|60000| idle connection check interval
-spring.datasource.timeBetweenConnectErrorMillis|60000| retry interval
-spring.datasource.minEvictableIdleTimeMillis|300000| connections over minEvictableIdleTimeMillis will be collect when idle check
-spring.datasource.validationQuery|SELECT 1| validate connection by running the SQL
-spring.datasource.validationQueryTimeout|3| validate connection timeout[seconds]
-spring.datasource.testWhileIdle|true| set whether the pool validates the allocated connection when a new connection request comes
-spring.datasource.testOnBorrow|true| validity check when the program requests a new connection
-spring.datasource.testOnReturn|false| validity check when the program recalls a connection
-spring.datasource.defaultAutoCommit|true| whether auto commit
-spring.datasource.keepAlive|true| runs validationQuery SQL to avoid the connection closed by pool when the connection idles over minEvictableIdleTimeMillis
-spring.datasource.poolPreparedStatements|true| open PSCache
-spring.datasource.maxPoolPreparedStatementPerConnectionSize|20| specify the size of PSCache on each connection
 
+|Service| Configuration file  |
+|--|--|
+|Master Server | `master-server/conf/application.yaml`|
+|Api Server| `api-server/conf/application.yaml`|
+|Worker Server| `worker-server/conf/application.yaml`|
+|Alert Server| `alert-server/conf/application.yaml`|
 
-### zookeeper.properties [zookeeper config properties]
+The default configuration is as follows:
 
 |Parameters | Default value| Description|
 |--|--|--|
-zookeeper.quorum|localhost:2181| ZooKeeper cluster connection info
+|spring.datasource.driver-class-name| org.postgresql.Driver |datasource driver|
-zookeeper.dolphinscheduler.root|/dolphinscheduler| DS is stored under ZooKeeper root directory
+|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |datasource connection url|
-zookeeper.session.timeout|60000|  session timeout
+|spring.datasource.username|root|datasource username|
-zookeeper.connection.timeout|30000| connection timeout
+|spring.datasource.password|root|datasource password|
-zookeeper.retry.base.sleep|100| time to wait between subsequent retries
+|spring.datasource.hikari.connection-test-query|select 1|validate connection by running the SQL|
-zookeeper.retry.max.sleep|30000| maximum time to wait between subsequent retries
+|spring.datasource.hikari.minimum-idle| 5| minimum connection pool size number|
-zookeeper.retry.maxtime|10| maximum retry times
+|spring.datasource.hikari.auto-commit|true|whether auto commit|
+|spring.datasource.hikari.pool-name|DolphinScheduler|name of the connection pool|
+|spring.datasource.hikari.maximum-pool-size|50| maximum connection pool size number|
+|spring.datasource.hikari.connection-timeout|30000|connection timeout|
+|spring.datasource.hikari.idle-timeout|600000|Maximum idle connection survival time|
+|spring.datasource.hikari.leak-detection-threshold|0|Connection leak detection threshold|
+|spring.datasource.hikari.initialization-fail-timeout|1|Connection pool initialization failed timeout|
 
+Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`.
+
+
+### Zookeeper related configuration
+DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions. Configuration file location:
+|Service| Configuration file  |
+|--|--|
+|Master Server | `master-server/conf/application.yaml`|
+|Api Server| `api-server/conf/application.yaml`|
+|Worker Server| `worker-server/conf/application.yaml`|
+
+The default configuration is as follows:
+
+|Parameters | Default value| Description|
+|--|--|--|
+|registry.zookeeper.namespace|dolphinscheduler|namespace of zookeeper|
+|registry.zookeeper.connect-string|localhost:2181| the connection string of zookeeper|
+|registry.zookeeper.retry-policy.base-sleep-time|60ms|time to wait between subsequent retries|
+|registry.zookeeper.retry-policy.max-sleep|300ms|maximum time to wait between subsequent retries|
+|registry.zookeeper.retry-policy.max-retries|5|maximum retry times|
+|registry.zookeeper.session-timeout|30s|session timeout|
+|registry.zookeeper.connection-timeout|30s|connection timeout|
+|registry.zookeeper.block-until-connected|600ms|waiting time to block until the connection succeeds|
+|registry.zookeeper.digest|{username}:{password}|digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) |
+
+Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
 
 ### common.properties [hadoop、s3、yarn config properties]
 
 Currently, common.properties mainly configures Hadoop,s3a related configurations.
-|Parameters | Default value| Description|
+| Parameters | Default value | Description |
 |--|--|--|
-data.basedir.path|/tmp/dolphinscheduler| local directory used to store temp files
+data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files
-resource.storage.type|NONE| type of resource files: HDFS, S3, NONE
+resource.storage.type | NONE | type of resource files: HDFS, S3, NONE
-resource.upload.path|/dolphinscheduler| storage path of resource files
+resource.storage.upload.base.path | /dolphinscheduler | storage path of resource files
-hadoop.security.authentication.startup.state|false| whether hadoop grant kerberos permission
+resource.aws.access.key.id | minioadmin | access key id of S3
-java.security.krb5.conf.path|/opt/krb5.conf|kerberos config directory
+resource.aws.secret.access.key | minioadmin | secret access key of S3
-login.user.keytab.username|hdfs-mycluster@ESZ.COM|kerberos username
+resource.aws.region |us-east-1 | region of S3
-login.user.keytab.path|/opt/hdfs.headless.keytab|kerberos user keytab
+resource.aws.s3.bucket.name | dolphinscheduler | bucket name of S3
-kerberos.expire.time|2|kerberos expire time,integer,the unit is hour
+resource.aws.s3.endpoint | http://minio:9000 | endpoint of S3
-resource.view.suffixs| txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties| file types supported by resource center
+resource.hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS
-hdfs.root.user|hdfs| configure users with corresponding permissions if storage type is HDFS
+resource.hdfs.fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory
-fs.defaultFS|hdfs://mycluster:8020|If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory
+hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission
-fs.s3a.endpoint||s3 endpoint url
+java.security.krb5.conf.path | /opt/krb5.conf | kerberos config directory
-fs.s3a.access.key||s3 access key
+login.user.keytab.username | hdfs-mycluster@ESZ.COM | kerberos username
-fs.s3a.secret.key||s3 secret key
+login.user.keytab.path | /opt/hdfs.headless.keytab | kerberos user keytab
-yarn.resourcemanager.ha.rm.ids||specify the yarn resourcemanager url. if resourcemanager supports HA, input HA IP addresses (separated by comma), or input null for standalone
+kerberos.expire.time | 2 | kerberos expire time,integer,the unit is hour
-yarn.application.status.address|http://ds1:8088/ws/v1/cluster/apps/%s|keep default if ResourceManager supports HA or not use ResourceManager, or replace ds1 with corresponding hostname if ResourceManager in standalone mode
+yarn.resourcemanager.ha.rm.ids |  | specify the yarn resourcemanager url. if resourcemanager supports HA, input HA IP addresses (separated by comma), or input null for standalone
-dolphinscheduler.env.path|env/dolphinscheduler_env.sh|load environment variables configs [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
+yarn.application.status.address | http://ds1:8088/ws/v1/cluster/apps/%s | keep default if ResourceManager supports HA or not use ResourceManager, or replace ds1 with corresponding hostname if ResourceManager in standalone mode
-development.state|false| specify whether in development state
+dolphinscheduler.env.path | env/dolphinscheduler_env.sh | load environment variables configs [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
+development.state | false | specify whether in development state
+task.resource.limit.state | false | specify whether in resource limit state
 
 
 ### application-api.properties [API-service log config]
 
-|Parameters | Default value| Description|
+|Service| Configuration file  |
+|--|--|
+|Master Server | `master-server/conf/common.properties`|
+|Api Server| `api-server/conf/common.properties`|
+|Worker Server| `worker-server/conf/common.properties`|
+|Alert Server| `alert-server/conf/common.properties`|
+
+The default configuration is as follows:
+
+| Parameters | Default value | Description |
 |--|--|--|
-server.port|12345|api service communication port
+|data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files|
-server.servlet.session.timeout|7200|session timeout
+|resource.storage.type | NONE | type of resource files: HDFS, S3, NONE|
-server.servlet.context-path|/dolphinscheduler | request path
+|resource.upload.path | /dolphinscheduler | storage path of resource files|
-spring.servlet.multipart.max-file-size|1024MB| maximum file size
+|aws.access.key.id | minioadmin | access key id of S3|
-spring.servlet.multipart.max-request-size|1024MB| maximum request size
+|aws.secret.access.key | minioadmin | secret access key of S3|
-server.jetty.max-http-post-size|5000000| jetty maximum post size
+|aws.region | us-east-1 | region of S3|
-spring.messages.encoding|UTF-8| message encoding
+|aws.s3.endpoint | http://minio:9000 | endpoint of S3|
-spring.jackson.time-zone|GMT+8| time zone
+|hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS|
-spring.messages.basename|i18n/messages| i18n config
+|fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory|
-security.authentication.type|PASSWORD| authentication type
+|hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission|
+|java.security.krb5.conf.path | /opt/krb5.conf | kerberos config directory|
+|login.user.keytab.username | hdfs-mycluster@ESZ.COM | kerberos username|
+|login.user.keytab.path | /opt/hdfs.headless.keytab | kerberos user keytab|
+|kerberos.expire.time | 2 | kerberos expire time,integer,the unit is hour|
+|yarn.resourcemanager.ha.rm.ids | 192.168.xx.xx,192.168.xx.xx | specify the yarn resourcemanager url. if resourcemanager supports HA, input HA IP addresses (separated by comma), or input null for standalone|
+|yarn.application.status.address | http://ds1:8088/ws/v1/cluster/apps/%s | keep default if ResourceManager supports HA or not use ResourceManager, or replace ds1 with corresponding hostname if ResourceManager in standalone mode|
+|development.state | false | specify whether in development state|
+|dolphin.scheduler.network.interface.preferred | NONE | display name of the network card|
+|dolphin.scheduler.network.priority.strategy | default | IP acquisition strategy, give priority to finding the internal network or the external network|
+|resource.manager.httpaddress.port | 8088 | the port of resource manager|
+|yarn.job.history.status.address | http://ds1:19888/ws/v1/history/mapreduce/jobs/%s | job history status url of yarn|
+|datasource.encryption.enable | false | whether to enable datasource encryption|
+|datasource.encryption.salt | !@#$%^&* | the salt of the datasource encryption|
+|data-quality.jar.name | dolphinscheduler-data-quality-dev-SNAPSHOT.jar | the jar of data quality|
+|support.hive.oneSession | false | specify whether hive SQL is executed in the same session|
+|sudo.enable | true | whether to enable sudo|
+|alert.rpc.port | 50052 | the RPC port of Alert Server|
+|zeppelin.rest.url | http://localhost:8080 | the RESTful API url of zeppelin|
 
 
-### master.properties [master-service log config]
+### Api-server related configuration
+Location: `api-server/conf/application.yaml`
 
 |Parameters | Default value| Description|
 |--|--|--|
-master.listen.port|5678|master listen port
+|server.port|12345|api service communication port|
-master.exec.threads|100|master-service execute thread number, used to limit the number of process instances in parallel
+|server.servlet.session.timeout|120m|session timeout|
-master.exec.task.num|20|defines the number of parallel tasks for each process instance of the master-service
+|server.servlet.context-path|/dolphinscheduler/ |request path|
-master.dispatch.task.num|3|defines the number of dispatch tasks for each batch of the master-service
+|spring.servlet.multipart.max-file-size|1024MB|maximum file size|
-master.host.selector|LowerWeight|master host selector, to select a suitable worker to run the task, optional value: random, round-robin, lower weight
+|spring.servlet.multipart.max-request-size|1024MB|maximum request size|
-master.heartbeat.interval|10|master heartbeat interval, the unit is second
+|server.jetty.max-http-post-size|5000000|jetty maximum post size|
-master.task.commit.retryTimes|5|master commit task retry times
+|spring.banner.charset|UTF-8|message encoding|
-master.task.commit.interval|1000|master commit task interval, the unit is millisecond
+|spring.jackson.time-zone|UTC|time zone|
-master.max.cpuload.avg|-1|master max CPU load avg, only higher than the system CPU load average, master server can schedule. default value -1: the number of CPU cores * 2
+|spring.jackson.date-format|"yyyy-MM-dd HH:mm:ss"|time format|
-master.reserved.memory|0.3|master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
+|spring.messages.basename|i18n/messages|i18n config|
+|security.authentication.type|PASSWORD|authentication type|
+|security.authentication.ldap.user.admin|read-only-admin|admin user account when you log-in with LDAP|
+|security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls|
+|security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn|
+|security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username|
+|security.authentication.ldap.password|password|LDAP password|
+|security.authentication.ldap.user.identity.attribute|uid|LDAP user identity attribute|
+|security.authentication.ldap.user.email.attribute|mail|LDAP user email attribute|
+|traffic.control.global.switch|false|traffic control global switch|
+|traffic.control.max-global-qps-rate|300|global max request number per second|
+|traffic.control.tenant-switch|false|traffic control tenant switch|
+|traffic.control.default-tenant-qps-rate|10|default tenant max request number per second|
+|traffic.control.customize-tenant-qps-rate||customize tenant max request number per second|
 
-
+### Master Server related configuration
-### worker.properties [worker-service log config]
+Location: `master-server/conf/application.yaml`
 
 |Parameters | Default value| Description|
 |--|--|--|
-worker.listen.port|1234|worker-service listen port
+|master.listen-port|5678|master listen port|
-worker.exec.threads|100|worker-service execute thread number, used to limit the number of task instances in parallel
+|master.fetch-command-num|10|the number of commands fetched by master|
-worker.heartbeat.interval|10|worker-service heartbeat interval, the unit is second
+|master.pre-exec-threads|10|master prepare execute thread number to limit handle commands in parallel|
-worker.max.cpuload.avg|-1|worker max CPU load avg, only higher than the system CPU load average, worker server can be dispatched tasks. default value -1: the number of CPU cores * 2
+|master.exec-threads|100|master execute thread number to limit process instances in parallel|
-worker.reserved.memory|0.3|worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
+|master.dispatch-task-number|3|master dispatch task number per batch|
-worker.groups|default|worker groups separated by comma, e.g., 'worker.groups=default,test' <br> worker will join corresponding group according to this config when startup
+|master.host-selector|lower_weight|master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight|
+|master.heartbeat-interval|10|master heartbeat interval, the unit is second|
+|master.task-commit-retry-times|5|master commit task retry times|
+|master.task-commit-interval|1000|master commit task interval, the unit is millisecond|
+|master.state-wheel-interval|5|time to check status|
+|master.max-cpu-load-avg|-1|master max CPU load avg, only higher than the system CPU load average, master server can schedule. default value -1: the number of CPU cores * 2|
+|master.reserved-memory|0.3|master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G|
+|master.failover-interval|10|failover interval, the unit is minute|
+|master.kill-yarn-job-when-task-failover|true|whether to kill yarn job when failover taskInstance|
 
 
-### alert.properties [alert-service log config]
+### Worker Server related configuration
+Location: `worker-server/conf/application.yaml`
 
 |Parameters | Default value| Description|
 |--|--|--|
-alert.type|EMAIL|alter type|
+|worker.listen-port|1234|worker-service listen port|
-mail.protocol|SMTP|mail server protocol
+|worker.exec-threads|100|worker-service execute thread number, used to limit the number of task instances in parallel|
-mail.server.host|xxx.xxx.com|mail server host
+|worker.heartbeat-interval|10|worker-service heartbeat interval, the unit is second|
-mail.server.port|25|mail server port
+|worker.host-weight|100|worker host weight to dispatch tasks|
-mail.sender|xxx@xxx.com|mail sender email
+|worker.tenant-auto-create|true|tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.|
-mail.user|xxx@xxx.com|mail sender email name
+|worker.max-cpu-load-avg|-1|worker max CPU load avg, only higher than the system CPU load average, worker server can be dispatched tasks. default value -1: the number of CPU cores * 2|
-mail.passwd|111111|mail sender email password
+|worker.reserved-memory|0.3|worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G|
-mail.smtp.starttls.enable|true|specify mail whether open tls
+|worker.groups|default|worker groups separated by comma, e.g., 'worker.groups=default,test' <br> worker will join corresponding group according to this config when startup|
-mail.smtp.ssl.enable|false|specify mail whether open ssl
+|worker.alert-listen-host|localhost|the alert listen host of worker|
-mail.smtp.ssl.trust|xxx.xxx.com|specify mail ssl trust list
+|worker.alert-listen-port|50052|the alert listen port of worker|
-xls.file.path|/tmp/xls|mail attachment temp storage directory
+
-||following configure WeCom[optional]|
+### Alert Server related configuration
-enterprise.wechat.enable|false|specify whether enable WeCom
+Location: `alert-server/conf/application.yaml`
-enterprise.wechat.corp.id|xxxxxxx|WeCom corp id
+
-enterprise.wechat.secret|xxxxxxx|WeCom secret
+|Parameters | Default value| Description|
-enterprise.wechat.agent.id|xxxxxxx|WeCom agent id
+|--|--|--|
-enterprise.wechat.users|xxxxxxx|WeCom users
+|server.port|50053|the port of Alert Server|
-enterprise.wechat.token.url|https://qyapi.weixin.qq.com/cgi-bin/gettoken?  <br /> corpid=$corpId&corpsecret=$secret|WeCom token url
+|alert.port|50052|the port of alert|
-enterprise.wechat.push.url|https://qyapi.weixin.qq.com/cgi-bin/message/send?  <br /> access_token=$token|WeCom push url
-enterprise.wechat.user.send.msg||send message format
-enterprise.wechat.team.send.msg||group message format
-plugin.dir|/Users/xx/your/path/to/plugin/dir|plugin directory
 
 
-### quartz.properties [quartz config properties]
+### Quartz related configuration
 
 This part describes quartz configs and configure them based on your practical situation and resources.
-|Parameters | Default value| Description|
-|--|--|--|
-org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.StdJDBCDelegate |
-org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate |
-org.quartz.scheduler.instanceName | DolphinScheduler |
-org.quartz.scheduler.instanceId | AUTO |
-org.quartz.scheduler.makeSchedulerThreadDaemon | true |
-org.quartz.jobStore.useProperties | false |
-org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool |
-org.quartz.threadPool.makeThreadsDaemons | true |
-org.quartz.threadPool.threadCount | 25 |
-org.quartz.threadPool.threadPriority | 5 |
-org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX |
-org.quartz.jobStore.tablePrefix | QRTZ_ |
-org.quartz.jobStore.isClustered | true |
-org.quartz.jobStore.misfireThreshold | 60000 |
-org.quartz.jobStore.clusterCheckinInterval | 5000 |
-org.quartz.jobStore.acquireTriggersWithinLock|true |
-org.quartz.jobStore.dataSource | myDs |
-org.quartz.dataSource.myDs.connectionProvider.class | org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider |
 
+|Service| Configuration file  |
+|--|--|
+|Master Server | `master-server/conf/application.yaml`|
+|Api Server| `api-server/conf/application.yaml`|
 
-### install_config.conf [DS environment variables configuration script[install or start DS]]
+The default configuration is as follows:
 
-install_config.conf is a bit complicated and is mainly used in the following two places.
+|Parameters | Default value|
-* DS Cluster Auto Installation.
+|--|--|
+|spring.quartz.properties.org.quartz.threadPool.threadPriority | 5|
+|spring.quartz.properties.org.quartz.jobStore.isClustered | true|
+|spring.quartz.properties.org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX|
+|spring.quartz.properties.org.quartz.scheduler.instanceId | AUTO|
+|spring.quartz.properties.org.quartz.jobStore.tablePrefix | QRTZ_|
+|spring.quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock|true|
+|spring.quartz.properties.org.quartz.scheduler.instanceName | DolphinScheduler|
+|spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool|
+|spring.quartz.properties.org.quartz.jobStore.useProperties | false|
+|spring.quartz.properties.org.quartz.threadPool.makeThreadsDaemons | true|
+|spring.quartz.properties.org.quartz.threadPool.threadCount | 25|
+|spring.quartz.properties.org.quartz.jobStore.misfireThreshold | 60000|
+|spring.quartz.properties.org.quartz.scheduler.makeSchedulerThreadDaemon | true|
+|spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate|
+|spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000|
 
-> System will load configs in the install_config.conf and auto-configure files below, based on the file content when executing 'install.sh'.
-> Files such as dolphinscheduler-daemon.sh, datasource.properties, zookeeper.properties, common.properties, application-api.properties, master.properties, worker.properties, alert.properties, quartz.properties, etc.
-
-* Startup and Shutdown DS Cluster.
-
-> The system will load masters, workers, alert-server, API-servers and other parameters inside the file to startup or shutdown DS cluster.
-
-#### File Content
-
-```bash
-
-# Note:  please escape the character if the file contains special characters such as `.*[]^${}\+?|()@#&`.
-#   eg: `[` escape to `\[`
-
-# Database type (DS currently only supports PostgreSQL and MySQL)
-dbtype="mysql"
-
-# Database url and port
-dbhost="192.168.xx.xx:3306"
-
-# Database name
-dbname="dolphinscheduler"
-
-
-# Database username
-username="xx"
-
-# Database password
-password="xx"
-
-# ZooKeeper url
-zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-
-# DS installation path, such as '/data1_1T/dolphinscheduler'
-installPath="/data1_1T/dolphinscheduler"
-
-# Deployment user
-# Note: Deployment user needs 'sudo' privilege and has rights to operate HDFS.
-#     Root directory must be created by the same user if using HDFS, otherwise permission related issues will be raised.
-deployUser="dolphinscheduler"
-
-
-# Followings are alert-service configs
-# Mail server host
-mailServerHost="smtp.exmail.qq.com"
-
-# Mail server port
-mailServerPort="25"
-
-# Mail sender
-mailSender="xxxxxxxxxx"
-
-# Mail user
-mailUser="xxxxxxxxxx"
-
-# Mail password
-mailPassword="xxxxxxxxxx"
-
-# Whether mail supports TLS
-starttlsEnable="true"
-
-# Whether mail supports SSL. Note: starttlsEnable and sslEnable cannot both set true.
-sslEnable="false"
-
-# Mail server host, same as mailServerHost
-sslTrust="smtp.exmail.qq.com"
-
-# Specify which resource upload function to use for resources storage, such as sql files. And supported options are HDFS, S3 and NONE. HDFS for upload to HDFS and NONE for not using this function.
-resourceStorageType="NONE"
-
-# if S3, write S3 address. HA, for example: s3a://dolphinscheduler，
-# Note: s3 make sure to create the root directory /dolphinscheduler
-defaultFS="hdfs://mycluster:8020"
-
-# If parameter 'resourceStorageType' is S3, following configs are needed:
-s3Endpoint="http://192.168.xx.xx:9010"
-s3AccessKey="xxxxxxxxxx"
-s3SecretKey="xxxxxxxxxx"
-
-# If ResourceManager supports HA, then input master and standby node IP or hostname, eg: '192.168.xx.xx,192.168.xx.xx'. Or else ResourceManager run in standalone mode, please set yarnHaIps="" and "" for not using yarn.
-yarnHaIps="192.168.xx.xx,192.168.xx.xx"
-
-
-# If ResourceManager runs in standalone, then set ResourceManager node ip or hostname, or else remain default.
-singleYarnIp="yarnIp1"
-
-# Storage path when using HDFS/S3
-resourceUploadPath="/dolphinscheduler"
-
-
-# HDFS/S3 root user
-hdfsRootUser="hdfs"
-
-# Followings are Kerberos configs
-
-# Specify Kerberos enable or not
-kerberosStartUp="false"
-
-# Kdc krb5 config file path
-krb5ConfPath="$installPath/conf/krb5.conf"
-
-# Keytab username
-keytabUserName="hdfs-mycluster@ESZ.COM"
-
-# Username keytab path
-keytabPath="$installPath/conf/hdfs.headless.keytab"
-
-
-# API-service port
-apiServerPort="12345"
-
-
-# All hosts deploy DS
-ips="ds1,ds2,ds3,ds4,ds5"
-
-# Ssh port, default 22
-sshPort="22"
-
-# Master service hosts
-masters="ds1,ds2"
-
-# All hosts deploy worker service
-# Note: Each worker needs to set a worker group name and default name is "default"
-workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
-
-#  Host deploy alert-service
-alertServer="ds3"
-
-# Host deploy API-service
-apiServers="ds1"
-```
 
 ### dolphinscheduler_env.sh [load environment variables configs]
 
@@ -407,7 +364,6 @@ export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
 # Database related configuration, set database type, username and password
 export DATABASE=${DATABASE:-postgresql}
 export SPRING_PROFILES_ACTIVE=${DATABASE}
-export SPRING_DATASOURCE_DRIVER_CLASS_NAME
 export SPRING_DATASOURCE_URL
 export SPRING_DATASOURCE_USERNAME
 export SPRING_DATASOURCE_PASSWORD
@@ -434,11 +390,11 @@ export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
 export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
 ```
 
-### Services logback configs
+### Log related configuration
 
-Services name| logback config name |
+|Service| Configuration file  |
---|--|
+|--|--|
-API-service logback config |logback-api.xml|
+|Master Server | `master-server/conf/logback-spring.xml`|
-master-service logback config|logback-master.xml |
+|Api Server| `api-server/conf/logback-spring.xml`|
-worker-service logback config|logback-worker.xml |
+|Worker Server| `worker-server/conf/logback-spring.xml`|
-alert-service logback config|logback-alert.xml |
+|Alert Server| `alert-server/conf/logback-spring.xml`|

docs/docs/en/architecture/design.md

@@ -5,7 +5,7 @@
 ### System Architecture Diagram
 
 <p align="center">
-  <img src="/img/architecture-1.3.0.jpg" alt="System architecture diagram"  width="70%" />
+  <img src="../../../img/architecture-lastest.jpg" alt="System architecture diagram"  width="70%" />
   <p align="center">
         <em>System architecture diagram</em>
   </p>
@@ -14,7 +14,7 @@
 ### Start Process Activity Diagram
 
 <p align="center">
-  <img src="/img/process-start-flow-1.3.0.png" alt="Start process activity diagram"  width="70%" />
+  <img src="../../../img/process-start-flow-1.3.0.png" alt="Start process activity diagram"  width="70%" />
   <p align="center">
         <em>Start process activity diagram</em>
   </p>
@@ -29,25 +29,35 @@
     MasterServer provides monitoring services based on netty.
 
     #### The Service Mainly Includes:
+  
+    - **DistributedQuartz** distributed scheduling component, which is mainly responsible for the start and stop operations of scheduled tasks. When quartz start the task, there will be a thread pool inside the Master responsible for the follow-up operation of the processing task;
 
-    - **Distributed Quartz** distributed scheduling component, which is mainly responsible for the start and stop operations of schedule tasks. When Quartz starts the task, there will be a thread pool inside the Master responsible for the follow-up operation of the processing task.
+    - **MasterSchedulerService** is a scanning thread that regularly scans the `t_ds_command` table in the database, runs different business operations according to different **command types**;
 
-    - **MasterSchedulerThread** is a scanning thread that regularly scans the **command** table in the database and runs different business operations according to different **command types**.
+    - **WorkflowExecuteRunnable** is mainly responsible for DAG task segmentation, task submission monitoring, and logical processing of different event types;
 
-    - **MasterExecThread** is mainly responsible for DAG task segmentation, task submission monitoring, and logical processing to different command types.
+    - **TaskExecuteRunnable** is mainly responsible for the processing and persistence of tasks, and generates task events and submits them to the event queue of the process instance;
 
-    - **MasterTaskExecThread** is mainly responsible for the persistence to tasks.
+    - **EventExecuteService** is mainly responsible for the polling of the event queue of the process instances;
+
+    - **StateWheelExecuteThread** is mainly responsible for process instance and task timeout, task retry, task-dependent polling, and generates the corresponding process instance or task event and submits it to the event queue of the process instance;
+
+    - **FailoverExecuteThread** is mainly responsible for the logic of Master fault tolerance and Worker fault tolerance;
 
 * **WorkerServer** 
 
      WorkerServer also adopts a distributed and decentralized design concept. WorkerServer is mainly responsible for task execution and providing log services.
 
      When the WorkerServer service starts, register a temporary node with ZooKeeper and maintain a heartbeat.
-     Server provides monitoring services based on netty.
+     WorkerServer provides monitoring services based on netty.
   
      #### The Service Mainly Includes:
-  
+
-     - **Fetch TaskThread** is mainly responsible for continuously getting tasks from the **Task Queue**, and calling **TaskScheduleThread** corresponding executor according to different task types.
+    - **WorkerManagerThread** is mainly responsible for the submission of the task queue, continuously receives tasks from the task queue, and submits them to the thread pool for processing;
+
+    - **TaskExecuteThread** is mainly responsible for the process of task execution, and the actual processing of tasks according to different task types;
+
+    - **RetryReportTaskStatusThread** is mainly responsible for regularly polling to report the task status to the Master until the Master replies to the status ack to avoid the loss of the task status;
 
 * **ZooKeeper** 
 
@@ -55,18 +65,13 @@
 
     We have also implemented queues based on Redis, but we hope DolphinScheduler depends on as few components as possible, so we finally removed the Redis implementation.
 
-* **Task Queue** 
+* **AlertServer** 
 
-    Provide task queue operation, the current queue is also implement base on ZooKeeper. Due to little information stored in the queue, there is no need to worry about excessive data in the queue. In fact, we have tested the millions of data storage in queues, which has no impact on system stability and performance.
+  Provides alarm services, and implements rich alarm methods through alarm plugins.
-
-* **Alert** 
-
-    Provide alarm related interface, the interface mainly includes **alarm** two types of alarm data storage, query and notification functions. Among them, there are **email notification** and **SNMP (not yet implemented)**.
 
 * **API** 
 
     The API interface layer is mainly responsible for processing requests from the front-end UI layer. The service uniformly provides RESTful APIs to provide request services to external.
-    Interfaces include workflow creation, definition, query, modification, release, logoff, manual start, stop, pause, resume, start execution from specific node, etc.
 
 * **UI** 
 
@@ -102,41 +107,6 @@ Problems in centralized thought design:
 - In fact, truly decentralized distributed systems are rare. Instead, dynamic centralized distributed systems are constantly pouring out. Under this architecture, the managers in the cluster are dynamically selected, rather than preset, and when the cluster fails, the nodes of the cluster will automatically hold "meetings" to elect new "managers" To preside over the work. The most typical case is Etcd implemented by ZooKeeper and Go language.
 - The decentralization of DolphinScheduler is that the Master and Worker register in ZooKeeper, for implement the centerless feature to Master cluster and Worker cluster. Use the ZooKeeper distributed lock to elect one of the Master or Worker as the "manager" to perform the task.
 
-#### Distributed Lock Practice
-
-DolphinScheduler uses ZooKeeper distributed lock to implement only one Master executes Scheduler at the same time, or only one Worker executes the submission of tasks.
-1. The following shows the core process algorithm for acquiring distributed locks:
- <p align="center">
-   <img src="https://analysys.github.io/easyscheduler_docs_cn/images/distributed_lock.png" alt="Obtain distributed lock process"  width="50%" />
- </p>
-
-2. Flow diagram of implementation of Scheduler thread distributed lock in DolphinScheduler:
- <p align="center">
-   <img src="/img/distributed_lock_procss.png" alt="Obtain distributed lock process"  width="50%" />
- </p>
-
-
-#### Insufficient Thread Loop Waiting Problem
-
--  If there is no sub-process in a DAG, when the number of data in the Command is greater than the threshold set by the thread pool, the process directly waits or fails.
--  If a large DAG nests many sub-processes, there will produce a "dead" state as the following figure:
-
- <p align="center">
-   <img src="https://analysys.github.io/easyscheduler_docs_cn/images/lack_thread.png" alt="Insufficient threads waiting loop problem"  width="50%" />
- </p>
-In the above figure, MainFlowThread waits for the end of SubFlowThread1, SubFlowThread1 waits for the end of SubFlowThread2, SubFlowThread2 waits for the end of SubFlowThread3, and SubFlowThread3 waits for a new thread in the thread pool, then the entire DAG process cannot finish, and the threads cannot be released. In this situation, the state of the child-parent process loop waiting is formed. At this moment, unless a new Master is started and add threads to break such a "stalemate", the scheduling cluster will no longer use.
-
-It seems a bit unsatisfactory to start a new Master to break the deadlock, so we proposed the following three solutions to reduce this risk:
-
-1. Calculate the sum of all Master threads, and then calculate the number of threads required for each DAG, that is, pre-calculate before the DAG process executes. Because it is a multi-master thread pool, it is unlikely to obtain the total number of threads in real time. 
-2. Judge whether the single-master thread pool is full, let the thread fail directly when fulfilled.
-3. Add a Command type with insufficient resources. If the thread pool is insufficient, suspend the main process. In this way, there are new threads in the thread pool, which can make the process suspended by insufficient resources wake up to execute again.
-
-Note: The Master Scheduler thread executes by FIFO when acquiring the Command.
-
-So we choose the third way to solve the problem of insufficient threads.
-
-
 #### Fault-Tolerant Design
 
 Fault tolerance divides into service downtime fault tolerance and task retry, and service downtime fault tolerance divides into master fault tolerance and worker fault tolerance.
@@ -155,7 +125,7 @@ Among them, the Master monitors the directories of other Masters and Workers. If
 - Master fault tolerance：
 
 <p align="center">
-   <img src="/img/failover-master.jpg" alt="failover-master"  width="50%" />
+   <img src="../../../img/failover-master.jpg" alt="failover-master"  width="50%" />
  </p>
 
 Fault tolerance range: From the perspective of host, the fault tolerance range of Master includes: own host and node host that does not exist in the registry, and the entire process of fault tolerance will be locked;
@@ -167,7 +137,7 @@ Fault-tolerant post-processing: After the fault tolerance of ZooKeeper Master co
 - Worker fault tolerance：
 
 <p align="center">
-   <img src="/img/failover-worker.jpg" alt="failover-worker"  width="50%" />
+   <img src="../../../img/failover-worker.jpg" alt="failover-worker"  width="50%" />
  </p>
 
 Fault tolerance range: From the perspective of process instance, each Master is only responsible for fault tolerance of its own process instance; it will lock only when `handleDeadServer`;
@@ -188,11 +158,11 @@ Here we must first distinguish the concepts of task failure retry, process failu
 
 Next to the main point, we divide the task nodes in the workflow into two types.
 
-- One is a business node, which corresponds to an actual script or process command, such as shell node, MR node, Spark node, and dependent node.
+- One is a business task, which corresponds to an actual script or process command, such as Shell task, SQL task, and Spark task.
 
-- Another is a logical node, which does not operate actual script or process command, but only logical processing to the entire process flow, such as sub-process sections.
+- Another is a logical task, which does not operate actual script or process command, but only logical processing to the entire process flow, such as sub-process task, dependent task.
 
-Each **business node** can configure the number of failed retries. When the task node fails, it will automatically retry until it succeeds or exceeds the retry times. **Logical node** failure retry is not supported, but the tasks in the logical node support.
+**Business node** can configure the number of failed retries. When the task node fails, it will automatically retry until it succeeds or exceeds the retry times. **Logical node** failure retry is not supported.
 
 If there is a task failure in the workflow that reaches the maximum retry times, the workflow will fail and stop, and the failed workflow can be manually re-run or process recovery operations.
 
@@ -225,55 +195,29 @@ In the early schedule design, if there is no priority design and use the fair sc
    <img src="https://analysys.github.io/easyscheduler_docs_cn/images/grpc.png" alt="grpc remote access"  width="50%" />
  </p>
 
-- We use the customized FileAppender and Filter functions from Logback to implement each task instance generates one log file.
+- For details, please refer to the logback configuration of Master and Worker, as shown in the following example:
-- The following is the FileAppender implementation：
 
-```java
+```xml
- /**
+<conversionRule conversionWord="messsage" converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
-  * task log appender
+<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
-  */
+    <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
- public class TaskLogAppender extends FileAppender<ILoggingEvent> {
+    <Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
- 
+        <key>taskAppId</key>
-     ...
+        <logBase>${log.base}</logBase>
-
+    </Discriminator>
-    @Override
+    <sift>
-    protected void append(ILoggingEvent event) {
+        <appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender">
-
+            <file>${log.base}/${taskAppId}.log</file>
-        if (currentlyActiveFile == null){
+            <encoder>
-            currentlyActiveFile = getFile();
+                <pattern>
-        }
+                            [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %messsage%n
-        String activeFile = currentlyActiveFile;
+                </pattern>
-        // thread name： taskThreadName-processDefineId_processInstanceId_taskInstanceId
+                <charset>UTF-8</charset>
-        String threadName = event.getThreadName();
+            </encoder>
-        String[] threadNameArr = threadName.split("-");
+            <append>true</append>
-        // logId = processDefineId_processInstanceId_taskInstanceId
+        </appender>
-        String logId = threadNameArr[1];
+    </sift>
-        ...
+</appender>
-        super.subAppend(event);
-    }
-}
-```
-
-Generate logs in the form of /process definition id /process instance id /task instance id.log
-
-- Filter to match the thread name starting with TaskLogInfo:
-
-- The following shows the TaskLogFilter implementation:
-
- ```java
- /**
- *  task log filter
- */
-public class TaskLogFilter extends Filter<ILoggingEvent> {
-
-    @Override
-    public FilterReply decide(ILoggingEvent event) {
-        if (event.getThreadName().startsWith("TaskLogInfo-")){
-            return FilterReply.ACCEPT;
-        }
-        return FilterReply.DENY;
-    }
-}
 ```
 
 ## Sum Up

docs/docs/en/architecture/metadata.md

@@ -1,33 +1,7 @@
 # MetaData
 
-## DolphinScheduler DB Table Overview
+## Table Schema
-
+see sql files in `dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql`
-| Table Name | Comment |
-| :---: | :---: |
-| t_ds_access_token | token for access DolphinScheduler backend |
-| t_ds_alert | alert detail |
-| t_ds_alertgroup | alert group |
-| t_ds_command | command detail |
-| t_ds_datasource | data source |
-| t_ds_error_command | error command detail |
-| t_ds_process_definition | process definition |
-| t_ds_process_instance | process instance |
-| t_ds_project | project |
-| t_ds_queue | queue |
-| t_ds_relation_datasource_user | datasource related to user |
-| t_ds_relation_process_instance | sub process |
-| t_ds_relation_project_user | project related to user |
-| t_ds_relation_resources_user | resource related to user |
-| t_ds_relation_udfs_user | UDF functions related to user |
-| t_ds_relation_user_alertgroup | alert group related to user |
-| t_ds_resources | resoruce center file |
-| t_ds_schedules | process definition schedule |
-| t_ds_session | user login session |
-| t_ds_task_instance | task instance |
-| t_ds_tenant | tenant |
-| t_ds_udfs | UDF resource |
-| t_ds_user | user detail |
-| t_ds_version | DolphinScheduler version |
 
 
 ---
@@ -36,158 +10,31 @@
 
 ### User Queue DataSource
 
-![image.png](/img/metadata-erd/user-queue-datasource.png)
+![image.png](../../../img/metadata-erd/user-queue-datasource.png)
 
 - One tenant can own Multiple users.
-- The queue field in the t_ds_user table stores the queue_name information in the t_ds_queue table, t_ds_tenant stores queue information using queue_id column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue.
+- The queue field in the `t_ds_user` table stores the `queue_name` information in the `t_ds_queue` table, `t_ds_tenant` stores queue information using `queue_id` column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue.
-- The user_id field in the t_ds_datasource table shows the user who create the data source. The user_id in t_ds_relation_datasource_user shows the user who has permission to the data source.
+- The `user_id` field in the `t_ds_datasource` table shows the user who create the data source. The user_id in `t_ds_relation_datasource_user` shows the user who has permission to the data source.
   
 ### Project Resource Alert
 
-![image.png](/img/metadata-erd/project-resource-alert.png)
+![image.png](../../../img/metadata-erd/project-resource-alert.png)
 
-- User can have multiple projects, user project authorization completes the relationship binding using project_id and user_id in t_ds_relation_project_user table.
+- User can have multiple projects, user project authorization completes the relationship binding using `project_id` and `user_id` in `t_ds_relation_project_user` table.
-- The user_id in the t_ds_projcet table represents the user who create the project, and the user_id in the t_ds_relation_project_user table represents users who have permission to the project.
+- The `user_id` in the `t_ds_projcet` table represents the user who create the project, and the `user_id` in the `t_ds_relation_project_user` table represents users who have permission to the project.
-- The user_id in the t_ds_resources table represents the user who create the resource, and the user_id in t_ds_relation_resources_user represents the user who has permissions to the resource.
+- The `user_id` in the `t_ds_resources` table represents the user who create the resource, and the `user_id` in `t_ds_relation_resources_user` represents the user who has permissions to the resource.
-- The user_id in the t_ds_udfs table represents the user who create the UDF, and the user_id in the t_ds_relation_udfs_user table represents a user who has permission to the UDF.
+- The `user_id` in the `t_ds_udfs` table represents the user who create the UDF, and the `user_id` in the `t_ds_relation_udfs_user` table represents a user who has permission to the UDF.
-  
-### Command Process Task
 
-![image.png](/img/metadata-erd/command.png)<br />![image.png](/img/metadata-erd/process-task.png)
+### Project - Tenant - ProcessDefinition - Schedule
+![image.png](../../../img/metadata-erd/project_tenant_process_definition_schedule.png)
 
-- A project has multiple process definitions, a process definition can generate multiple process instances, and a process instance can generate multiple task instances.
+- A project can have multiple process definitions, and each process definition belongs to only one project.
-- The t_ds_schedulers table stores the specified time schedule information for process definition.
+- A tenant can be used by multiple process definitions, and each process definition must select only one tenant.
-- The data stored in the t_ds_relation_process_instance table is used to deal with the sub-processes of a process definition, parent_process_instance_id field represents the id of the main process instance who contains child processes, process_instance_id field represents the id of the sub-process instance, parent_task_instance_id field represents the task instance id of the sub-process node.
+- A workflow definition can have one or more schedules.
-- The process instance table and the task instance table correspond to the t_ds_process_instance table and the t_ds_task_instance table, respectively.
 
----
+### Process Definition Execution
+![image.png](../../../img/metadata-erd/process_definition.png)
 
-## Core Table Schema
+- A process definition corresponds to multiple task definitions, which are associated through `t_ds_process_task_relation` and the associated key is `code + version`. When the pre-task of the task is empty, the corresponding `pre_task_node` and `pre_task_version` are 0.
-
+- A process definition can have multiple process instances `t_ds_process_instance`, one process instance corresponds to one or more task instances `t_ds_task_instance`.
-### t_ds_process_definition
+- The data stored in the `t_ds_relation_process_instance` table is used to handle the case that the process definition contains sub-processes. `parent_process_instance_id` represents the id of the main process instance containing the sub-process, `process_instance_id` represents the id of the sub-process instance, `parent_task_instance_id` represents the task instance id of the sub-process node. The process instance table and the task instance table correspond to the `t_ds_process_instance` table and the `t_ds_task_instance` table, respectively.
-
-| Field | Type | Comment |
-| --- | --- | --- |
-| id | int | primary key |
-| name | varchar | process definition name |
-| version | int | process definition version |
-| release_state | tinyint | process definition release state：0:offline,1:online |
-| project_id | int | project id |
-| user_id | int | process definition creator id |
-| process_definition_json | longtext | process definition JSON content |
-| description | text | process definition description |
-| global_params | text | global parameters |
-| flag | tinyint | whether process available: 0 not available, 1 available |
-| locations | text | Node location information |
-| connects | text | Node connection information |
-| receivers | text | receivers |
-| receivers_cc | text | carbon copy list |
-| create_time | datetime | create time |
-| timeout | int | timeout |
-| tenant_id | int | tenant id |
-| update_time | datetime | update time |
-| modify_by | varchar | define user modify the process |
-| resource_ids | varchar | resource id set |
-
-### t_ds_process_instance
-
-| Field | Type | Comment |
-| --- | --- | --- |
-| id | int | primary key |
-| name | varchar | process instance name |
-| process_definition_id | int | process definition id |
-| state | tinyint | process instance Status: 0 successful commit, 1 running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6 fail, 7 succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait for dependency to complete |
-| recovery | tinyint | process instance failover flag：0: normal,1: failover instance needs restart |
-| start_time | datetime | process instance start time |
-| end_time | datetime | process instance end time |
-| run_times | int | process instance run times |
-| host | varchar | process instance host |
-| command_type | tinyint | command type：0 start ,1 start from the current node,2 resume a fault-tolerant process,3 resume from pause process, 4 execute from the failed node,5 complement, 6 dispatch, 7 re-run, 8 pause, 9 stop, 10 resume waiting thread |
-| command_param | text | JSON command parameters |
-| task_depend_type | tinyint | node dependency type: 0 current node, 1 forward, 2 backward |
-| max_try_times | tinyint | max try times |
-| failure_strategy | tinyint | failure strategy, 0: end the process when node failed,1: continue run the other nodes when failed |
-| warning_type | tinyint | warning type 0: no warning, 1: warning if process success, 2: warning if process failed, 3: warning whatever results |
-| warning_group_id | int | warning group id |
-| schedule_time | datetime | schedule time |
-| command_start_time | datetime | command start time |
-| global_params | text | global parameters |
-| process_instance_json | longtext | process instance JSON |
-| flag | tinyint | whether process instance is available: 0 not available, 1 available |
-| update_time | timestamp | update time |
-| is_sub_process | int | whether the process is sub process: 1 sub-process, 0 not sub-process |
-| executor_id | int | executor id |
-| locations | text | node location information |
-| connects | text | node connection information |
-| history_cmd | text | history commands, record all the commands to a instance |
-| dependence_schedule_times | text | depend schedule estimate time |
-| process_instance_priority | int | process instance priority. 0 highest,1 high,2 medium,3 low,4 lowest |
-| worker_group | varchar | worker group who assign the task |
-| timeout | int | timeout |
-| tenant_id | int | tenant id |
-
-### t_ds_task_instance
-
-| Field | Type | Comment |
-| --- | --- | --- |
-| id | int | primary key |
-| name | varchar | task name |
-| task_type | varchar | task type |
-| process_definition_id | int | process definition id |
-| process_instance_id | int | process instance id |
-| task_json | longtext | task content JSON |
-| state | tinyint | Status: 0 commit succeeded, 1 running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6 fail, 7 succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait for dependency to complete |
-| submit_time | datetime | task submit time |
-| start_time | datetime | task start time |
-| end_time | datetime | task end time |
-| host | varchar | host of task running on |
-| execute_path | varchar | task execute path in the host |
-| log_path | varchar | task log path |
-| alert_flag | tinyint | whether alert |
-| retry_times | int | task retry times |
-| pid | int | pid of task |
-| app_link | varchar | Yarn app id |
-| flag | tinyint | task instance is available : 0 not available, 1 available |
-| retry_interval | int | retry interval when task failed |
-| max_retry_times | int | max retry times |
-| task_instance_priority | int | task instance priority:0 highest,1 high,2 medium,3 low,4 lowest |
-| worker_group | varchar | worker group who assign the task |
-
-#### t_ds_schedules
-
-| Field | Type | Comment |
-| --- | --- | --- |
-| id | int | primary key |
-| process_definition_id | int | process definition id |
-| start_time | datetime | schedule start time |
-| end_time | datetime | schedule end time |
-| crontab | varchar | crontab expression |
-| failure_strategy | tinyint | failure strategy: 0 end,1 continue |
-| user_id | int | user id |
-| release_state | tinyint | release status: 0 not yet released,1 released |
-| warning_type | tinyint | warning type: 0: no warning, 1: warning if process success, 2: warning if process failed, 3: warning whatever results |
-| warning_group_id | int | warning group id |
-| process_instance_priority | int | process instance priority:0 highest,1 high,2 medium,3 low,4 lowest |
-| worker_group | varchar | worker group who assign the task |
-| create_time | datetime | create time |
-| update_time | datetime | update time |
-
-### t_ds_command
-
-| Field | Type | Comment |
-| --- | --- | --- |
-| id | int | primary key |
-| command_type | tinyint | command type: 0 start workflow, 1 start execution from current node, 2 resume fault-tolerant workflow, 3 resume pause process, 4 start execution from failed node, 5 complement, 6 schedule, 7 re-run, 8 pause, 9 stop, 10 resume waiting thread |
-| process_definition_id | int | process definition id |
-| command_param | text | JSON command parameters |
-| task_depend_type | tinyint | node dependency type: 0 current node, 1 forward, 2 backward |
-| failure_strategy | tinyint | failed policy: 0 end, 1 continue |
-| warning_type | tinyint | alarm type: 0 no alarm, 1 alarm if process success, 2: alarm if process failed, 3: warning whatever results |
-| warning_group_id | int | warning group id |
-| schedule_time | datetime | schedule time |
-| start_time | datetime | start time |
-| executor_id | int | executor id |
-| dependence | varchar | dependence column |
-| update_time | datetime | update time |
-| process_instance_priority | int | process instance priority: 0 highest,1 high,2 medium,3 low,4 lowest |
-| worker_group_id | int |  worker group who assign the task |

docs/docs/en/contribute/api-standard.md

@@ -20,25 +20,25 @@ Use URI to locate the resource, and use GET to indicate query.
 + When the URI is a type of resource, it means to query a type of resource. For example, the following example indicates paging query `alter-groups`.
 ```
 Method: GET
-/api/dolphinscheduler/alert-groups
+/dolphinscheduler/alert-groups
 ```
 
 + When the URI is a single resource, it means to query this resource. For example, the following example means to query the specified `alter-group`.
 ```
 Method: GET
-/api/dolphinscheduler/alter-groups/{id}
+/dolphinscheduler/alter-groups/{id}
 ```
 
 + In addition, we can also express query sub-resources based on URI, as follows:
 ```
 Method: GET
-/api/dolphinscheduler/projects/{projectId}/tasks
+/dolphinscheduler/projects/{projectId}/tasks
 ```
 
 **The above examples all represent paging query. If we need to query all data, we need to add `/list` after the URI to distinguish. Do not mix the same API for both paged query and query.**
 ```
 Method: GET
-/api/dolphinscheduler/alert-groups/list
+/dolphinscheduler/alert-groups/list
 ```
 
 ### ② Create - POST
@@ -48,13 +48,13 @@ Use URI to locate the resource, use POST to indicate create, and then return the
 
 ```
 Method: POST
-/api/dolphinscheduler/alter-groups
+/dolphinscheduler/alter-groups
 ```
 
 + create sub-resources is also the same as above.
 ```
 Method: POST
-/api/dolphinscheduler/alter-groups/{alterGroupId}/tasks
+/dolphinscheduler/alter-groups/{alterGroupId}/tasks
 ```
 
 ### ③ Modify - PUT
@@ -62,7 +62,7 @@ Use URI to locate the resource, use PUT to indicate modify.
 + modify an `alert-group`
 ```
 Method: PUT
-/api/dolphinscheduler/alter-groups/{alterGroupId}
+/dolphinscheduler/alter-groups/{alterGroupId}
 ```
 
 ### ④ Delete -DELETE
@@ -71,20 +71,28 @@ Use URI to locate the resource, use DELETE to indicate delete.
 + delete an `alert-group`
 ```
 Method: DELETE
-/api/dolphinscheduler/alter-groups/{alterGroupId}
+/dolphinscheduler/alter-groups/{alterGroupId}
 ```
 
 + batch deletion: batch delete the id array，we should use POST. **（Do not use the DELETE method, because the body of the DELETE request has no semantic meaning, and it is possible that some gateways, proxies, and firewalls will directly strip off the request body after receiving the DELETE request.）**
 ```
 Method: POST
-/api/dolphinscheduler/alter-groups/batch-delete
+/dolphinscheduler/alter-groups/batch-delete
 ```
 
-### ⑤ Others
+### ⑤ Partial Modifications -PATCH
+Use URI to locate the resource, use PATCH to partial modifications.
+
+```
+Method: PATCH
+/dolphinscheduler/alter-groups/{alterGroupId}
+```
+
+### ⑥ Others
 In addition to creating, deleting, modifying and quering, we also locate the corresponding resource through url, and then append operations to it after the path, such as:
 ```
-/api/dolphinscheduler/alert-groups/verify-name
+/dolphinscheduler/alert-groups/verify-name
-/api/dolphinscheduler/projects/{projectCode}/process-instances/{code}/view-gantt
+/dolphinscheduler/projects/{projectCode}/process-instances/{code}/view-gantt
 ```
 
 ## 3. Parameter design
@@ -94,7 +102,7 @@ In the case of paging, if the parameter entered by the user is less than 1, the
 
 ## 4. Others design
 ### base URL
-The URI of the project needs to use `/api/<project_name>` as the base path, so as to identify that these APIs are under this project.
+The URI of the project needs to use `/<project_name>` as the base path, so as to identify that these APIs are under this project.
 ```
-/api/dolphinscheduler
+/dolphinscheduler
 ```

docs/docs/en/contribute/api-test.md

@@ -0,0 +1,94 @@
+# DolphinScheduler API Automation Test
+
+## Preparatory knowledge
+
+### The difference between API Test and Unit Test
+
+API test, which imitates the user calling API, starts from a certain entry and performs operations step by step until a certain work is completed. Different from unit testing, the latter usually needs to test parameters, parameter types, parameter values, parameter numbers, return values, throw errors, etc. in order to ensure that a specific function can complete its work stably and reliably in any case. Unit tests assume that the entire product will work as long as all functions work properly.
+
+In contrast, API testing focuses on whether a complete operation chain can be completed
+
+For example, the API test of the tenant management interface focuses on whether users can log in normally; If the login fails, whether the error message can be displayed correctly. After logging in, you can perform tenant management operations through the sessionid you carry.
+
+
+## API Test
+
+### API-Pages
+
+DolphinScheduler's API tests are deployed using docker-compose. The current tests are in standalone mode and are mainly used to check some basic functions such as "add, delete, change and check". For further cluster validation, such as collaboration between services or communication mechanisms between services, refer to `deploy/docker/docker-compose.yml` for configuration.
+
+For API test,  the [page model](https://www.selenium.dev/documentation/guidelines/page_object_models/) form is used, mainly to create a corresponding model for each page. The following is an example of a login page.
+
+```java
+package org.apache.dolphinscheduler.api.test.pages;
+
+
+import org.apache.dolphinscheduler.api.test.entity.HttpResponse;
+import org.apache.dolphinscheduler.api.test.utils.RequestClient;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public final class LoginPage {
+    public HttpResponse login(String username, String password) {
+        Map<String, Object> params = new HashMap<>();
+
+        params.put("userName", username);
+        params.put("userPassword", password);
+
+        RequestClient requestClient = new RequestClient();
+
+        return requestClient.post("/login", null, params);
+    }
+}
+```
+
+During the test process, we only test the interfaces we need to focus on, not all interfaces in the page. Therefore, we only declare the user name, password and interface path on the login page.
+
+In addition, during the testing process, the interface are not requested directly. The general choice is to package the corresponding methods to achieve the effect of reuse. For example, if you want to log in, you input your username and password through the `public LoginPage login()` method to manipulate the elements you pass in to achieve the effect of logging in. That is, when the user finishes logging in, he or she achieve the effect of login.
+
+On the login page, only the input parameter specification of the interface request is defined. For the output parameter of the interface request, only the unified basic response structure is defined. The data actually returned by the interface is tested in the actual test case. Whether the input and output of main test interfaces can meet the requirements of test cases.
+
+
+### API-Cases
+
+The following is an example of a tenant management test. As explained earlier, we use docker-compose for deployment, so for each test case, we need to import the corresponding file in the form of an annotation.
+
+The interface is requested using the RemoteWebDriver provided with Selenium. Before each test case is started there is some preparation work that needs to be done. For example: logging in the user, jumping to the corresponding page (depending on the specific test case).
+
+```java
+    @BeforeAll
+    public static void setup() {
+        LoginPage loginPage = new LoginPage();
+        HttpResponse loginHttpResponse = loginPage.login(user, password);
+
+        sessionId = JSONUtils.convertValue(loginHttpResponse.body().data(), LoginResponseData.class).sessionId();
+    }
+```
+
+When the preparation is complete, it is time for the formal test case writing. We use a form of @Order() annotation for modularity, to confirm the order of the tests. After the tests have been run, assertions are used to determine if the tests were successful, and if the assertion returns true, the tenant creation was successful. The following code can be used as a reference:
+
+```java
+    @Test
+    @Order(1)
+    public void testCreateTenant() {
+        TenantPage tenantPage = new TenantPage();
+
+        HttpResponse createTenantHttpResponse = tenantPage.createTenant(sessionId, tenant, 1, "");
+
+        Assertions.assertTrue(createTenantHttpResponse.body().success());
+    }
+```
+
+The rest are similar cases and can be understood by referring to the specific source code.
+
+https://github.com/apache/dolphinscheduler/tree/dev/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases
+
+## Supplements
+
+When running API tests locally, First, you need to start the local service, you can refer to this page: 
+[development-environment-setup](./development-environment-setup.md)
+
+When running API tests locally, the `-Dlocal=true` parameter can be configured to connect locally and facilitate changes to the UI.
+
+The current default request timeout length is 10 seconds. This value should not be modified without special requirements.

docs/docs/en/contribute/architecture-design.md

@@ -97,9 +97,7 @@ Before explaining the architecture of the schedule system, let us first understa
 
      - **UI**
 
-       The front-end page of the system provides various visual operation interfaces of the system. For details, see the [quick start](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/about/introduction.html) section.
+    The front-end page of the system provides various visual operation interfaces of the system. For details, see the [quick start](https://dolphinscheduler.apache.org/en-us/docs/3.0.5/about/introduction) section.
-
-     
 
 #### 2.3 Architectural Design Ideas
 

docs/docs/en/contribute/development-environment-setup.md

@@ -1,14 +1,13 @@
 # DolphinScheduler development
 
-## Software Requests
+## Software Requirements
-
 Before setting up the DolphinScheduler development environment, please make sure you have installed the software as below:
 
-* [Git](https://git-scm.com/downloads): DolphinScheduler version control system
+* [Git](https://git-scm.com/downloads)
-* [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html): DolphinScheduler backend language
+* [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html): v1.8.x (Currently does not support jdk 11)
-* [Maven](http://maven.apache.org/download.cgi): Java Package Management System
+* [Maven](http://maven.apache.org/download.cgi): v3.5+
-* [Node](https://nodejs.org/en/download): DolphinScheduler frontend
+* [Node](https://nodejs.org/en/download): v16.13+ (dolphinScheduler version is lower than 3.0, please install node v12.20+)
- language
+* [Pnpm](https://pnpm.io/installation): v6.x
 
 ### Clone Git Repository
 
@@ -19,11 +18,65 @@ mkdir dolphinscheduler
 cd dolphinscheduler
 git clone git@github.com:apache/dolphinscheduler.git
 ```
+
 ### compile source code
 
-i. If you use MySQL database, pay attention to modify pom.xml in the root project, and change the scope of the mysql-connector-java dependency to compile.
+Supporting system:
+* MacOS
+* Liunx
 
-ii. Run `mvn clean install -Prelease -Dmaven.test.skip=true`
+Run `mvn clean install -Prelease -Dmaven.test.skip=true`
+
+## Docker image build
+
+DolphinScheduler will release new Docker images after it released, you could find them in [Docker Hub](https://hub.docker.com/search?q=DolphinScheduler).
+
+* If you want to modify DolphinScheduler source code, and build Docker images locally, you can run when finished the modification
+```shell
+cd dolphinscheduler
+./mvnw -B clean package \
+       -Dmaven.test.skip \
+       -Dmaven.javadoc.skip \
+       -Dmaven.checkstyle.skip \
+       -Ddocker.tag=<TAG> \
+       -Pdocker,release              
+```
+
+When the command is finished you could find them by command `docker imaegs`.
+
+* If you want to modify DolphinScheduler source code, build and push Docker images to your registry <HUB_URL>，you can run when finished the modification
+```shell
+cd dolphinscheduler
+./mvnw -B clean deploy \
+       -Dmaven.test.skip \
+       -Dmaven.javadoc.skip \
+       -Dmaven.checkstyle.skip \
+       -Dmaven.deploy.skip \
+       -Ddocker.tag=<TAG> \
+       -Ddocker.hub=<HUB_URL> \
+       -Pdocker,release           
+```
+
+* If you want to modify DolphinScheduler source code, and also want to add customize dependencies of Docker image, you can modify the definition of Dockerfile after modifying the source code. You can run the following command to find all Dockerfile files.
+
+```shell
+cd dolphinscheduler
+find . -iname 'Dockerfile'
+```
+
+Then run the Docker build command above
+
+* You could create custom Docker images base on those images if you want to change image like add some dependencies or upgrade package.
+
+```Dockerfile
+FROM dolphinscheduler-standalone-server
+RUN apt update ; \
+    apt install -y <YOUR-CUSTOM-DEPENDENCE> ; \
+```
+
+> **_Note：_** Docker will build and push linux/amd64,linux/arm64 multi-architecture images by default
+>
+> Have to use version after Docker 19.03, because after 19.03 docker contains buildx
 
 
 ## Notice
@@ -36,8 +89,10 @@ There are two ways to configure the DolphinScheduler development environment, st
 ## DolphinScheduler Standalone Quick Start
 
 > **_Note:_** Use standalone server only for development and debugging, because it uses H2 Database as default database and Zookeeper Testing Server which may not be stable in production.
+> 
 > Standalone is only supported in DolphinScheduler 1.3.9 and later versions.
-> Standalone server is able to connect to external databases like mysql and postgresql, see [Standalone Deployment](https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/installation/standalone.html) for instructions.
+>
+> Standalone server is able to connect to external databases like mysql and postgresql, see [Standalone Deployment](https://dolphinscheduler.apache.org/en-us/docs/3.0.5/guide/installation/standalone) for instructions.
 
 ### Git Branch Choose
 
@@ -48,11 +103,12 @@ Use different Git branch to develop different codes
 
 ### Start backend server
 
-Find the class `org.apache.dolphinscheduler.server.StandaloneServer` in Intellij IDEA and clikc run main function to startup.
+Find the class `org.apache.dolphinscheduler.StandaloneServer` in Intellij IDEA and clikc run main function to startup.
 
 ### Start frontend server
 
-Install frontend dependencies and run it
+Install frontend dependencies and run it.
+> Note: You can see more detail about the frontend setting in [frontend development](./frontend-development.md).
 
 ```shell
 cd dolphinscheduler-ui
@@ -94,36 +150,29 @@ Following steps will guide how to start the DolphinScheduler backend service
 ##### Backend Start Prepare
 
 * Open project: Use IDE open the project, here we use Intellij IDEA as an example, after opening it will take a while for Intellij IDEA to complete the dependent download
-* Plugin installation(**Only required for 2.0 or later**)
 
- * Registry plug-in configuration, take Zookeeper as an example (registry.properties)
-  dolphinscheduler-service/src/main/resources/registry.properties
-  ```registry.properties
-   registry.plugin.name=zookeeper
-   registry.servers=127.0.0.1:2181
-  ```
 * File change
   * If you use MySQL as your metadata database, you need to modify `dolphinscheduler/pom.xml` and change the `scope` of the `mysql-connector-java` dependency to `compile`. This step is not necessary to use PostgreSQL
-  * Modify database configuration, modify the database configuration in the `dolphinscheduler-dao/src/main/resources/application-mysql.yaml`
+  * Modify database configuration, modify the database configuration in the `dolphinscheduler-master/src/main/resources/application.yaml`
+  * Modify database configuration, modify the database configuration in the `dolphinscheduler-worker/src/main/resources/application.yaml`
+  * Modify database configuration, modify the database configuration in the `dolphinscheduler-api/src/main/resources/application.yaml`
 
 
-  We here use MySQL with database, username, password named dolphinscheduler as an example
+We here use MySQL with database, username, password named dolphinscheduler as an example
-  ```application-mysql.yaml
+  ```application.yaml
    spring:
      datasource:
-       driver-class-name: com.mysql.jdbc.Driver
+       driver-class-name: com.mysql.cj.jdbc.Driver
        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-       username: ds_user
+       username: dolphinscheduler
        password: dolphinscheduler
   ```
 
 * Log level: add a line `<appender-ref ref="STDOUT"/>` to the following configuration to enable the log to be displayed on the command line
 
-  `dolphinscheduler-server/src/main/resources/logback-worker.xml`
+  `dolphinscheduler-master/src/main/resources/logback-spring.xml`
-  
+  `dolphinscheduler-worker/src/main/resources/logback-spring.xml`
-  `dolphinscheduler-server/src/main/resources/logback-master.xml` 
+  `dolphinscheduler-api/src/main/resources/logback-spring.xml`
-  
-  `dolphinscheduler-api/src/main/resources/logback-api.xml` 
 
   here we add the result after modify as below:
 
@@ -141,9 +190,9 @@ Following steps will guide how to start the DolphinScheduler backend service
 
 There are three services that need to be started, including MasterServer, WorkerServer, ApiApplicationServer.
 
-* MasterServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.master.MasterServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
+* MasterServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.master.MasterServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-spring.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
-* WorkerServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.worker.WorkerServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
+* WorkerServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.worker.WorkerServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-spring.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
-* ApiApplicationServer：Execute function `main` in the class `org.apache.dolphinscheduler.api.ApiApplicationServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api,mysql`. After it started, you could find Open API documentation in http://localhost:12345/dolphinscheduler/doc.html
+* ApiApplicationServer：Execute function `main` in the class `org.apache.dolphinscheduler.api.ApiApplicationServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-spring.xml -Dspring.profiles.active=api,mysql`. After it started, you could find Open API documentation in http://localhost:12345/dolphinscheduler/doc.html
 
 > The `mysql` in the VM Options `-Dspring.profiles.active=mysql` means specified configuration file
 

docs/docs/en/contribute/e2e-test.md

@@ -74,7 +74,7 @@ During the test process, we only test the elements we need to focus on, not all
 
 In addition, during the testing process, the elements are not manipulated directly. The general choice is to package the corresponding methods to achieve the effect of reuse. For example, if you want to log in, you input your username and password through the `public TenantPage login()` method to manipulate the elements you pass in to achieve the effect of logging in. That is, when the user finishes logging in, he or she jumps to the Security Centre (which goes to the Tenant Management page by default).
 
-The goToTab method is provided in SecurityPage to test the corresponding sidebar jumps, which include TenantPage, UserPage and WorkerGroupPge and QueuePage. These pages are implemented in the same way, to test that the form's input, add and delete buttons return the corresponding pages.
+The SecurityPage provides goToTab methods to test the corresponding sidebar jumps, mainly including TenantPage, UserPage, WorkerGroupPage and QueuePage. These pages are implemented in the same way, mainly to test whether the input, add and delete buttons of the form can return to the corresponding page.
 
 ```java
  public <T extends SecurityPage.Tab> T goToTab(Class<T> tab) {
@@ -104,7 +104,7 @@ The goToTab method is provided in SecurityPage to test the corresponding sidebar
     }
 ```
 
-![SecurityPage](/img/e2e-test/SecurityPage.png)
+![SecurityPage](../../../img/e2e-test/SecurityPage.png)
 
 For navigation bar options jumping, the goToNav method is provided in `org/apache/dolphinscheduler/e2e/pages/common/NavBarPage.java`. The currently supported pages are: ProjectPage, SecurityPage and ResourcePage.
 
@@ -139,7 +139,7 @@ For navigation bar options jumping, the goToNav method is provided in `org/apach
 
 Current E2E test cases supported include: File Management, Project Management, Queue Management, Tenant Management, User Management, Worker Group Management and Workflow Test.
 
-![E2E_Cases](/img/e2e-test/E2E_Cases.png)
+![E2E_Cases](../../../img/e2e-test/E2E_Cases.png)
 
 The following is an example of a tenant management test. As explained earlier, we use docker-compose for deployment, so for each test case, we need to import the corresponding file in the form of an annotation.
 
@@ -179,19 +179,19 @@ https://github.com/apache/dolphinscheduler/tree/dev/dolphinscheduler-e2e/dolphin
 ##  III. Supplements
 
 When running E2E tests locally, First, you need to start the local service, you can refer to this page: 
-[development-environment-setup](https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html)
+[development-environment-setup](./development-environment-setup.md)
 
 When running E2E tests locally, the `-Dlocal=true` parameter can be configured to connect locally and facilitate changes to the UI.
 
 When running E2E tests with `M1` chip, you can use `-Dm1_chip=true` parameter to configure containers supported by 
 `ARM64`.
 
-![Dlocal](/img/e2e-test/Dlocal.png)
+![Dlocal](../../../img/e2e-test/Dlocal.png)
 
 If a connection timeout occurs during a local run, increase the load time to a recommended 30 and above.
 
-![timeout](/img/e2e-test/timeout.png)
+![timeout](../../../img/e2e-test/timeout.png)
 
 The test run will be available as an MP4 file.
 
-![MP4](/img/e2e-test/MP4.png)
+![MP4](../../../img/e2e-test/MP4.png)

docs/docs/en/contribute/have-questions.md

@@ -0,0 +1,28 @@
+# Have Questions?
+
+## Slack
+
+Chat rooms are great for quick questions or discussions on specialized topics.
+
+The following chat rooms are officially part of Apache DolphinScheduler:
+
+​	The Slack workspace URL: http://asf-dolphinscheduler.slack.com/.
+
+​	You can join through invitation url: https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-1e0s4agia-Ofkxa93zStmqTm9UsQBE8g.
+
+This chat room is used for questions and discussions related to using DolphinScheduler.
+
+## Mailing Lists
+
+- [dev@dolphinscheduler.apache.org](https://lists.apache.org/list.html?dev@dolphinscheduler.apache.org) is for people who want to contribute code to DolphinScheduler. [(subscribe)](mailto:dev-subscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20subscribe)) [(unsubscribe)](mailto:dev-unsubscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20unsubscribe)) [(archives)](http://lists.apache.org/list.html?dev@dolphinscheduler.apache.org)
+
+Some quick tips when using email:
+
+- Tagging the subject line of your email will help you get a faster response, e.g. [api-server]: How to get open api interface?
+
+- Tags may help identify a topic by:
+  - Component: MasterServer,ApiServer,WorkerServer,AlertServer, etc
+  - Level: Beginner, Intermediate, Advanced
+  - Scenario: Debug, How-to
+
+- For error logs or long code examples, please use [GitHub gist](https://gist.github.com/) and include only a few lines of the pertinent code / log within the email.

docs/docs/en/contribute/join/DS-License.md

@@ -0,0 +1,42 @@
+# License Notice
+
+As we know that DolphinScheduler is an open-source undergoing project at The Apache Software Foundation (ASF), which means that you have to follow the Apache way to become the DolphinScheduler contributor. Furthermore, Apache has extremely strict rules according to the License. This passage will explain the ASF license and how to avoid License risks at the early stage when you participate in DolphinScheduler.
+
+Note: This article only applies to the Apache projects.
+
+### Licenses Could be Accepted to the Apache Project
+
+You have to pay attention to the following open-source software protocols which Apache projects support when you intend to add a new feature to the DolphinScheduler (or other Apache projects), which functions refers to other open-source software references.
+
+[ASF 3RD PARTY LICENSE POLICY](https://apache.org/legal/resolved.html)
+
+If the 3rd party software is not present at the above policy, we are sorry that your code can not pass the audit and we suggest searching for other substitute plans.
+
+Besides,  when you demand new dependencies in the project, please email us about the reason and the outcome of the influence to dev@dolphinscheduler.apache.org to discuss. Besides, you need at least 3 positive votes from the PPMC to finish the whole step.
+
+### How to Legally Use 3rd Party Open-source Software in the DolphinScheduler
+
+Moreover, when we intend to refer a new software ( not limited to 3rd party jar, text, CSS, js, pics, icons, audios etc and modifications based on 3rd party files) to our project, we need to use them legally in addition to the permission of ASF. Refer to the following article:
+
+* [COMMUNITY-LED DEVELOPMENT "THE APACHE WAY"](https://apache.org/dev/licensing-howto.html)
+
+
+For example, we should contain the NOTICE file (every open-source project has NOTICE file, generally under root directory) of ZooKeeper in our project when we are using ZooKeeper. As the Apache explains, "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work.
+
+We are not going to dive into every 3rd party open-source license policy, you may look up them if interested.
+
+### DolphinScheduler-License Check Rules
+
+In general, we would have our License-check scripts to our project. DolphinScheduler-License is provided by [kezhenxu94](https://github.com/kezhenxu94) which differ a bit from other open-source projects. All in all, we are trying to make sure avoiding the license issues at the first time.
+
+We need to follow the following steps when we need to add new jars or external resources:
+
+* Add the name and the version of the jar file in the known-dependencies.txt
+* Add relevant maven repository address under 'dolphinscheduler-dist/release-docs/LICENSE' directory
+* Append relevant NOTICE files under 'dolphinscheduler-dist/release-docs/NOTICE' directory and make sure they are no different to the original repository
+* Add relevant source code protocols under 'dolphinscheduler-dist/release-docs/license/' directory and the file name should be named as license+filename.txt. Eg: license-zk.txt
+
+### References
+
+* [COMMUNITY-LED DEVELOPMENT "THE APACHE WAY"](https://apache.org/dev/licensing-howto.html)
+* [ASF 3RD PARTY LICENSE POLICY](https://apache.org/legal/resolved.html)

docs/docs/en/contribute/join/become-a-committer.md

@@ -0,0 +1,11 @@
+# How to Become DolphinScheduler Committer
+
+Anyone can be a contributor to an Apache project. Being a contributor simply means that you take an interest in the project and contribute in some way, ranging from asking sensible questions (which documents the project and provides feedback to developers) through to providing new features as patches.
+
+If you become a valuable contributor to the project you may well be invited to become a committer. Committer is a term used at the ASF to signify someone who is committed to a particular project. It brings with it the privilege of write access to the project repository and resources.
+
+In Dolphinscheduler community, if a committer who have earned even more merit, can be invited to be a part of the Project Management Committee (PMC).
+
+One thing that is sometimes hard to understand when you are new to the open development process used at the ASF, is that we value the community more than the code. A strong and healthy community will be respectful and be a fun and rewarding place. More importantly, a diverse and healthy community can continue to support the code over the longer term, even as individual companies come and go from the field.
+
+More details could be found [here](https://community.apache.org/contributors/).

docs/docs/en/contribute/join/code-conduct.md

@@ -0,0 +1,68 @@
+# Code of Conduct
+
+The following Code of Conduct is based on full compliance with the [Apache Software Foundation Code of Conduct](https://www.apache.org/foundation/policies/conduct.html).
+
+## Development philosophy
+ - **Consistent** code style, naming, and usage are consistent.  
+ - **Easy to read** code is obvious, easy to read and understand, when debugging one knows the intent of the code.
+ - **Neat** agree with the concepts of《Refactoring》and《Code Cleanliness》and pursue clean and elegant code.
+ - **Abstract** hierarchy is clear and the concepts are refined and reasonable. Keep methods, classes, packages, and modules at the same level of abstraction.
+ - **Heart** Maintain a sense of responsibility and continue to be carved in the spirit of artisans.
+ 
+## Development specifications
+
+ - Executing `mvn -U clean package -Prelease` can compile and test through all test cases. 
+ - The test coverage tool checks for no less than dev branch coverage.
+ - In the root directory, use Checkstyle to check your code for special reasons for violating validation rules. The template location is located at ds_check_style.xml.
+ - Follow the coding specifications.
+
+## Coding specifications
+
+ - Use linux line breaks.
+ - Indentation (including empty lines) is consistent with the last line.
+ - An empty line is required between the class declaration and the following variable or method.
+ - There should be no meaningless empty lines.
+ - Classes, methods, and variables should be named as the name implies and abbreviations should be avoided.
+ - Return value variables are named after `result`; `each` is used in loops to name loop variables; and `entry` is used in map instead of `each`.
+ - The cached exception is called `e`; Catch the exception and do nothing, and the exception is named `ignored`.
+ - Configuration Files are named in camelCase, and file names are lowercase with uppercase initial/starting letter.
+ - Code that requires comment interpretation should be as small as possible and interpreted by method name.
+ - `equals` and `==` In a conditional expression, the constant is left, the variable is on the right, and in the expression greater than less than condition, the variable is left and the constant is right.
+ - In addition to the abstract classes used for inheritance, try to design the class as `final`.
+ - Nested loops are as much a method as possible.
+ - The order in which member variables are defined and the order in which parameters are passed is consistent across classes and methods.
+ - Priority is given to the use of guard statements.
+ - Classes and methods have minimal access control.
+ - The private method used by the method should follow the method, and if there are multiple private methods, the writing private method should appear in the same order as the private method in the original method.
+ - Method entry and return values are not allowed to be `null`.
+ - The return and assignment statements of if else are preferred with the tri-objective operator.
+ - Priority is given to `LinkedList` and only use `ArrayList` if you need to get element values in the collection through the index.
+ - Collection types such as `ArrayList`，`HashMap` that may produce expansion must specify the initial size of the collection to avoid expansion.
+ - Logs and notes are always in English.
+ - Comments can only contain `javadoc`, `todo` and `fixme`.
+ - Exposed classes and methods must have javadoc, other classes and methods and methods that override the parent class do not require javadoc.
+
+## Unit test specifications
+
+ - Test code and production code are subject to the same code specifications.
+ - Unit tests are subject to AIR (Automatic, Independent, Repeatable) Design concept.
+   - Automatic: Unit tests should be fully automated, not interactive. Manual checking of output results is prohibited, `System.out`, `log`, etc. are not allowed, and must be verified with assertions. 
+   - Independent: It is prohibited to call each other between unit test cases and to rely on the order of execution. Each unit test can be run independently.
+   - Repeatable: Unit tests cannot be affected by the external environment and can be repeated. 
+ - Unit tests are subject to BCDE（Border, Correct, Design, Error) Design principles.
+   - Border (Boundary value test): The expected results are obtained by entering the boundaries of loop boundaries, special values, data order, etc.
+   - Correct (Correctness test): The expected results are obtained with the correct input.
+   - Design (Rationality Design): Design high-quality unit tests in combination with production code design.
+   - Error (Fault tolerance test): The expected results are obtained through incorrect input such as illegal data, abnormal flow, etc.
+ - If there is no special reason, the test needs to be fully covered.
+ - Each test case needs to be accurately asserted.
+ - Prepare the environment for code separation from the test code.
+ - Only jUnit `Assert`，hamcrest `CoreMatchers`，Mockito Correlation can use static import.
+ - Single-data assertions should use `assertTrue`，`assertFalse`，`assertNull` and `assertNotNull`.
+ - Multi-data assertions should use `assertThat`.
+ - Accurate assertion, try not to use `not`，`containsString` assertion.
+ - The true value of the test case should be named actualXXX, and the expected value should be named expectedXXX.
+ - Classes and Methods with `@Test` labels do not require javadoc.
+
+ - Public specifications.
+   - Each line is no longer than `200` in length, ensuring that each line is semantically complete for easy understanding.

docs/docs/en/contribute/join/commit-message.md

@@ -0,0 +1,94 @@
+# Commit Message Notice
+
+### Preface
+
+A good commit message can help other developers (or future developers) quickly understand the context of related changes, and can also help project managers determine whether the commit is suitable for inclusion in the release. But when we checked the commit logs of many open source projects, we found an interesting problem. Some developers have very good code quality, but the commit message record is rather confusing. When other contributors or learners are viewing the code, it can’t be intuitively understood through commit log.
+The purpose of the changes before and after the submission, as Peter Hutterer said：Re-establishing the context of a piece of code is wasteful. We can’t avoid it completely, so our efforts should go to reducing it as much as possible. Commit messages can do exactly that and as a result, a commit message shows whether a developer is a good collaborator. Therefore, DolphinScheduler developed the protocol in conjunction with other communities and official Apache documents.
+
+### Commit Message RIP
+
+#### 1：Clearly modify the content
+
+A commit message should clearly state what issues (bug fixes, function enhancements, etc.) the submission solves, so that other developers can better track the issues and clarify the optimization during the version iteration process.
+
+#### 2：Associate the corresponding Pull Request or Issue
+
+When our changes are large, the commit message should best be associated with the relevant Issue or Pull Request on GitHub, so that our developers can quickly understand the context of the code submission through the associated information when reviewing the code. If the current commit is for an issue, then the issue can be closed in the Footer section.
+
+#### 3：Unified format
+
+The formatted CommitMessage can help provide more historical information for quick browsing, and it can also generate a Change Log directly from commit.
+
+Commit message should include three parts: Header, Body and Footer. Among them, Header is required, Body and Footer can be omitted.
+
+##### Header
+
+The header part has only one line, including three fields: type (required), scope (optional), and subject (required).
+
+[DS-ISSUE number][type] subject
+
+(1) Type is used to indicate the category of commit, and only the following 7 types are allowed.
+
+- feat：New features
+- fix：Bug fixes
+- docs：Documentation
+- style： Format (does not affect changes in code operation)
+- refactor：Refactoring (It is not a new feature or a code change to fix a bug)
+- test：Add test
+- chore：Changes in the build process or auxiliary tools
+
+If the type is feat and fix, the commit will definitely appear in the change log. Other types (docs, chore, style, refactor, test) are not recommended.
+
+(2) Scope
+
+Scope is used to indicate the scope of commit impact, such as server, remote, etc. If there is no suitable scope, you can use \*.
+
+(3) subject
+
+Subject is a short description of the purpose of the commit, no more than 50 characters.
+
+##### Body
+
+The body part is a detailed description of this commit, which can be divided into multiple lines, and the line break will wrap with 72 characters to avoid automatic line wrapping affecting the appearance.
+
+Note the following points in the Body section:
+
+- Use the verb-object structure, note the use of present tense. For example, use change instead of changed or changes
+
+- Don't capitalize the first letter
+
+- The end of the sentence does not need a ‘.’ (period)
+
+##### Footer
+
+Footer only works in two situations
+
+(1) Incompatible changes
+
+If the current code is not compatible with the previous version, the Footer part starts with BREAKING CHANGE, followed by a description of the change, the reason for the change, and the migration method.
+
+(2) Close Issue
+
+If the current commit is for a certain issue, you can close the issue in the Footer section, or close multiple issues at once.
+
+##### For Example
+
+```
+[DS-001][docs-en] add commit message
+
+- commit message RIP
+- build some conventions
+- help the commit messages become clean and tidy
+- help developers and release managers better track issues
+  and clarify the optimization in the version iteration
+
+This closes #001
+```
+
+### Reference documents
+
+[Commit message format](https://cwiki.apache.org/confluence/display/GEODE/Commit+Message+Format)
+
+[On commit messages-Peter Hutterer](http://who-t.blogspot.com/2009/12/on-commit-messages.html)
+
+[RocketMQ Community Operation Conventions](https://mp.weixin.qq.com/s/LKM4IXAY-7dKhTzGu5-oug)

docs/docs/en/contribute/join/contribute.md

@@ -0,0 +1,40 @@
+# Participate in Contributing
+
+First of all, thank you very much for choosing and using DolphinScheduler, and welcome to join the DolphinScheduler family!
+
+We encourage any form of participation in the community that will eventually become Committer or PPMC Such as:
+* Problems will be encountered via github on the [issue](https://github.com/apache/dolphinscheduler/issues) form feedback out.
+* Answer the issue questions that others are asking.
+* Help improve the documentation.
+* Help your project add test cases.
+* Add comments to the code.
+* Submit a PR that fixes the bug or Feature.
+* Publish application case practice, scheduling process analysis, or technical articles related to scheduling.
+* Help promote DolphinScheduler, participate in technical conferences or meetup, sharing and more.
+
+Welcome to the contributing team and join open source starting with submitting your first PR.
+ - For example, add code comments or find "easy to fix" tags or some very simple issue (misspellings, etc.) and so on, first familiarize yourself with the submission process through the first simple PR.
+ 
+Note: Contributions are not limited to PR Only, but contribute to the development of the project.
+
+I'm sure you'll benefit from open source by participating in DolphinScheduler!
+
+### 1. Participate in documentation contributions.
+
+Refer to the [Submit Guide-Document Notice](./document.md)
+
+### 2. Participate in code contributions.
+
+Refer to the [Submit Guide-Issue Notice](./issue.md), [Submit Guide-Pull Request Notice](./pull-request.md), [Submit Guide-Commit Message Notice](./commit-message.md)
+
+### 3. How to pick up an Issue and submit a Pull Request.
+
+If you want to implement a Feature or fix a Bug. Please refer to the following:
+
+* All Bugs and the new Features are recommended and managed using the Issues Page.
+* If you want to develop a Feature, first reply to the Issue associated with that feature, indicating that you are currently working on it. And set yourself a "deadline" when to Submit the Feature, and add it in the reply comment.
+* It's a good idea to find a mentor (or an instructor) in the core contributors who gives immediate feedback on design and functional implementation.
+* You should create a new branch to start your work, to get the name of the branch refer to the [Submit Guide-Pull Request Notice](./pull-request.md). For example, if you want to complete the feature and submit Issue 111, your branch name should be feature-111. The feature name can be determined after discussion with the instructor.
+* When you're done, send a Pull Request to dolphinscheduler, please refer to the《[Submit Guide-Submit Pull Request Process](./submit-code.md)》
+
+If you want to submit a Pull Request to complete a Feature or fix a Bug, it is recommended that you start with the `good first issue`, `easy-to-fix` issues, complete a small function to submit, do not change too many files at a time, changing too many files will also put a lot of pressure on Reviewers, it is recommended to submit them through multiple Pull Requests, not all at once.

docs/docs/en/contribute/join/document.md

@@ -0,0 +1,62 @@
+# Documentation Notice
+
+Good documentation is critical for any type of software. Any contribution that can improve the DolphinScheduler documentation is welcome.
+
+###  Get the document project
+
+Documentation for the DolphinScheduler project is maintained in a separate [git repository](https://github.com/apache/dolphinscheduler-website).
+
+First you need to fork the document project into your own github repository, and then clone the document to your local computer.
+
+```
+git clone https://github.com/<your-github-user-name>/dolphinscheduler-website
+```
+
+### The document environment
+
+The DolphinScheduler website is supported by [docsite](https://github.com/chengshiwen/docsite-ext)
+
+Make sure that your node version is 10+, docsite does not yet support versions higher than 10.x.
+
+### Document build guide
+
+1. Run `npm install` in the root directory to install the dependencies.
+
+2. Run commands to collect resources 2.1.Run `export PROTOCOL_MODE=ssh` tells Git clone resource via SSH protocol instead of HTTPS protocol. 2.2.Run `./scripts/prepare_docs.sh` prepare all related resources, for more information you could see [how prepare script work](https://github.com/apache/dolphinscheduler-website/blob/master/HOW_PREPARE_WOKR.md).
+
+3. Run `npm run start` in the root directory to start a local server, you will see the website in 'http://localhost:8080'.
+
+4. Run `npm run build` to build source code into dist directory.
+
+5. Verify your change locally: `python -m SimpleHTTPServer 8000`, when your python version is 3 use :`python3 -m http.server 8000` instead.
+
+If the latest version of node is installed locally, consider using `nvm` to allow different versions of `node` to run on your computer.
+
+1. Refer to the [Instructions](http://nvm.sh) to install nvm.
+
+2. Run `nvm install v10.23.1` to install node v10.
+
+3. Run `nvm use v10.23.1` to switch the current working environment to node v10.
+
+Now you can run and build the website in your local environment.
+
+### The document specification
+
+1. ** Spaces are Required ** between Chinese characters and English or numbers and ** Spaces are not required ** between Chinese punctuation marks and English or numbers, to enhance the aesthetics and readability of the Chinese-English mix.
+
+2. It is recommended that you use "you" in general. Of course, you can use the term when necessary, such as when there is a warning prompt.
+
+### How to submit a document Pull Request
+
+1. Do not use "git add." to commit all changes.
+
+2. Simply push the changed files, for example:
+
+ * `*.md`
+ * `blog.js or docs.js or site.js`
+
+3. Submit the Pull Request to the **master** branch.
+
+### Reference to the documentation
+
+[Apache Flink Translation Specifications](https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications)

docs/docs/en/contribute/join/e2e-guide.md

@@ -0,0 +1,36 @@
+# DolphinScheduler E2E Test Contribution Guide
+
+The main purpose of E2E test is to verify the integration and data integrity of the system, and its components, by simulating real user scenarios. This makes it possible to extend the scope of testing and ensure the health and stability of the system. To a certain extent, the test workload and costs are reduced. In simple terms, E2E testing is about treating a program as a black box and simulating the access behaviour of a real system from the user's point of view to see if the input to the test (user behaviour/simulated data) gives the expected results. Therefore, the community decided to add E2E automated testing to DolphinScheduler.
+
+The current community E2E test has not yet reached full coverage, so this document has been written with the aim of guiding more partners to get involved.
+
+### How to find the corresponding ISSUE?
+
+The E2E test pages are currently divided into four pages: Project Management, Resource Center, DataSource, and Security Center.
+
+Contributors can find the task by going to GitHub, searching for [apace/dolphinscheduler](https://github.com/apache/dolphinscheduler), and then searching for `e2e test cases` in the [issue list](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue+is%3Aopen+e2e+test+cases). As shown below.
+
+![e2e-issue](../../../../img/contribute/join/e2e/e2e-issue.png)
+
+In each issue, we list the content to be tested and the expected results, which can be seen in the Description. Once you are on the current page, you can select the issue you are interested in, for example, to participate in the Security Center test, just leave a comment under the corresponding issue about the case you want to test.
+
+- Task status: If the test has been completed, it is considered finished and as a contributor you need to find the outstanding tests.
+- number: Test case serial number.
+- function module: Functional modules to be tested, a functional module containing multiple test cases.
+- test point: Specific examples of what needs to be tested. For example button click actions on pages, page jump functions.
+- priority: The priority of the test case, **which is recommended to find a high priority**.
+- service: The service that needs to be started during the test process.
+- test steps: The test steps to be performed for each test case.
+- expected results: The expected test results.
+- actual results: The actual results.
+- remarks: The precautions needed during the test process.
+
+![e2e-security](../../../../img/contribute/join/e2e/e2e-security.png)
+
+### How to write test code?
+
+After taking up the corresponding task, the next step is to get to the crux of writing the code. Many partners may not be familiar with E2E test codes, so you can refer to this page: [e2e-test](../e2e-test.md).
+
+### How to submit a Pull Request?
+
+Participation in the open source community can take many forms, including issue, pull request and translation. To participate in the E2E testing process, contributors are first required to understand the simple process of submitting a pull request, see: [Pull Request](./pull-request.md).

docs/docs/en/contribute/join/issue.md

@@ -0,0 +1,136 @@
+# Issue Notice
+
+## Preface
+Issues function is used to track various Features, Bugs, Functions, etc. The project maintainer can organize the tasks to be completed through issues.
+
+Issue is an important step in drawing out a feature or bug,
+and the contents that can be discussed in an issue are not limited to the features, the causes of the existing bugs, the research on preliminary scheme, and the corresponding implementation design and code design.
+
+And only when the Issue is approved, the corresponding Pull Request should be implemented.
+
+If an issue corresponds to a large feature, it is recommended to divide it into multiple small issues according to the functional modules and other dimensions.
+
+## Specification
+
+### Issue title
+
+Title Format: [`Issue Type`][`Module Name`] `Issue Description`
+
+The `Issue Type` is as follows:
+
+<table>
+    <thead>
+        <tr>
+            <th style="width: 10%; text-align: center;">Issue Type</th>
+            <th style="width: 20%; text-align: center;">Description</th>
+            <th style="width: 20%; text-align: center;">Example</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td style="text-align: center;">Feature</td>
+            <td style="text-align: center;">Include expected new features and functions</td>
+            <td style="text-align: center;">[Feature][api] Add xxx api in xxx controller</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Bug</td>
+            <td style="text-align: center;">Bugs in the program</td>
+            <td style="text-align: center;">[Bug][api] Throw exception when xxx</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Improvement</td>
+            <td style="text-align: center;">Some improvements of the current program, not limited to code format, program performance, etc</td>
+            <td style="text-align: center;">[Improvement][server] Improve xxx between Master and Worker</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Test</td>
+            <td style="text-align: center;">Specifically for the test case</td>
+            <td style="text-align: center;">[Test][server] Add xxx e2e test</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Sub-Task</td>
+            <td style="text-align: center;">Those generally are subtasks of feature class. For large features, they can be divided into many small subtasks to complete one by one</td>
+            <td style="text-align: center;">[Sub-Task][server] Implement xxx in xxx</td>
+        </tr>
+    </tbody>
+</table>
+
+The `Module Name` is as follows:
+
+<table>
+    <thead>
+        <tr>
+            <th style="width: 10%; text-align: center;">Module Name</th>
+            <th style="width: 20%; text-align: center;">Description</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td style="text-align: center;">alert</td>
+            <td style="text-align: center;">Alert module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">api</td>
+            <td style="text-align: center;">Application program interface layer module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">service</td>
+            <td style="text-align: center;">Application service layer module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">dao</td>
+            <td style="text-align: center;">Application data access layer module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">plugin</td>
+            <td style="text-align: center;">Plugin module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">remote</td>
+            <td style="text-align: center;">Communication module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">server</td>
+            <td style="text-align: center;">Server module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">ui</td>
+            <td style="text-align: center;">Front end module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">docs-zh</td>
+            <td style="text-align: center;">Chinese document module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">docs</td>
+            <td style="text-align: center;">English document module</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">...</td>
+            <td style="text-align: center;">-</td>
+        </tr>
+    </tbody>
+</table>
+
+### Issue content template
+
+https://github.com/apache/dolphinscheduler/tree/dev/.github/ISSUE_TEMPLATE
+
+### Contributor
+
+Except for some special cases, it is recommended to discuss under issue or mailing list to determine the design scheme or provide the design scheme,
+as well as the code implementation design before completing the issue.
+
+If there are many different solutions, it is suggested to make a decision through mailing list or voting under issue.
+The issue can be implemented after final scheme and code implementation design being approved.
+The main purpose of this is to avoid wasting time caused by different opinions on implementation design or reconstruction in the pull request review stage.
+
+### Question
+
+- How to deal with the user who raises an issue does not know the module corresponding to the issue.
+
+    It is true that most users when raising issue do not know which module the issue belongs to.
+    In fact, this is very common in many open source communities. In this case, the committer / contributor actually knows the module affected by the issue.
+    If the issue is really valuable after being approved by committer and contributor, then the committer can modify the issue title according to the specific module involved in the issue,
+    or leave a message to the user who raises the issue to modify it into the corresponding title.
+

docs/docs/en/contribute/join/microbench.md

@@ -0,0 +1,100 @@
+# Micro BenchMark Notice
+
+All optimization must be based on data verification, and blind optimization is rejected. Based on this, we provide the MicroBench module.
+
+The MicroBench module is based on the OpenJDK JMH component (HotSpot's recommended benchmark test program). When you start benchmarking, you don't need additional dependencies.
+
+JMH, the Java MicroBenchmark Harness, is a tool suite dedicated to code microbenchmark testing. What is Micro Benchmark? Simply put, it is based on method-level benchmark testing, with an accuracy of microseconds. When you locate a hot method and want to further optimize the performance of the method, you can use JMH to quantitatively analyze the optimized results.
+
+### Several points to note in Java benchmark testing:
+
+- Prevent useless code from entering the test method.
+
+- Concurrent testing.
+
+- The test results are presented.
+
+### Typical application scenarios of JMH are:
+
+- 1: Quantitatively analyze the optimization effect of a hotspot function
+
+- 2: Want to quantitatively know how long a function needs to be executed, and the correlation between execution time and input variables
+
+- 3: Compare multiple implementations of a function
+
+DolphinScheduler-MicroBench provides AbstractBaseBenchmark, you can inherit from it, write your benchmark code, AbstractMicroBenchmark can guarantee to run in JUnit mode.
+
+### Customized operating parameters
+
+The default AbstractMicrobenchmark configuration is
+
+Warmup times 10 (warmupIterations)
+
+Number of tests 10 (measureIterations)
+
+Fork quantity 2 (forkCount)
+
+You can specify these parameters at startup，-DmeasureIterations, -DperfReportDir (output benchmark test result file directory), -DwarmupIterations, -DforkCount
+
+### DolphinScheduler-MicroBench Introduction
+
+It is generally not recommended to use fewer cycles when running tests. However, a smaller number of tests helps to verify the work during the benchmark test. After the verification is over, run a large number of benchmark tests.
+
+```java
+@Warmup(iterations = 2, time = 1)
+@Measurement(iterations = 4, time = 1)
+@State(Scope.Benchmark)
+public class EnumBenchMark extends AbstractBaseBenchmark {
+
+}
+```
+
+This can run benchmarks at the method level or the class level. Command line parameters will override the parameters on the annotation.
+
+```java
+@Benchmark // Method annotation, indicating that the method is an object that needs to be benchmarked.
+@BenchmarkMode(Mode.AverageTime) // Optional benchmark test mode is obtained through enumeration
+@OutputTimeUnit(TimeUnit.MICROSECONDS) // Output time unit
+public void enumStaticMapTest() {
+    TestTypeEnum.newGetNameByType(testNum);
+}
+```
+
+When your benchmark test is written, you can run it to view the specific test conditions: (The actual results depend on your system configuration)
+
+First, it will warm up our code,
+
+```java
+# Warmup Iteration   1: 0.007 us/op
+# Warmup Iteration   2: 0.008 us/op
+Iteration   1: 0.004 us/op
+Iteration   2: 0.004 us/op
+Iteration   3: 0.004 us/op
+Iteration   4: 0.004 us/op
+```
+
+After warmup, we usually get the following results
+
+```java
+Benchmark                        (testNum)   Mode  Cnt          Score           Error  Units
+EnumBenchMark.simpleTest               101  thrpt    8  428750972.826 ±  66511362.350  ops/s
+EnumBenchMark.simpleTest               108  thrpt    8  299615240.337 ± 290089561.671  ops/s
+EnumBenchMark.simpleTest               103  thrpt    8  288423221.721 ± 130542990.747  ops/s
+EnumBenchMark.simpleTest               104  thrpt    8  236811792.152 ± 155355935.479  ops/s
+EnumBenchMark.simpleTest               105  thrpt    8  472247775.246 ±  45769877.951  ops/s
+EnumBenchMark.simpleTest               103  thrpt    8  455473025.252 ±  61212956.944  ops/s
+EnumBenchMark.enumStaticMapTest        101   avgt    8          0.006 ±         0.003  us/op
+EnumBenchMark.enumStaticMapTest        108   avgt    8          0.005 ±         0.002  us/op
+EnumBenchMark.enumStaticMapTest        103   avgt    8          0.006 ±         0.005  us/op
+EnumBenchMark.enumStaticMapTest        104   avgt    8          0.006 ±         0.004  us/op
+EnumBenchMark.enumStaticMapTest        105   avgt    8          0.004 ±         0.001  us/op
+EnumBenchMark.enumStaticMapTest        103   avgt    8          0.004 ±         0.001  us/op
+EnumBenchMark.enumValuesTest           101   avgt    8          0.011 ±         0.004  us/op
+EnumBenchMark.enumValuesTest           108   avgt    8          0.025 ±         0.016  us/op
+EnumBenchMark.enumValuesTest           103   avgt    8          0.019 ±         0.010  us/op
+EnumBenchMark.enumValuesTest           104   avgt    8          0.018 ±         0.018  us/op
+EnumBenchMark.enumValuesTest           105   avgt    8          0.014 ±         0.012  us/op
+EnumBenchMark.enumValuesTest           103   avgt    8          0.012 ±         0.009  us/op
+```
+
+OpenJDK officially gave a lot of sample codes, interested students can query and learn JMH by themselves:[OpenJDK-JMH-Example](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/)

docs/docs/en/contribute/join/pull-request.md

@@ -0,0 +1,94 @@
+# Pull Request Notice
+
+## Preface
+Pull Request is a way of software cooperation, which is a process of bringing code involving different functions into the trunk. During this process, the code can be discussed, reviewed, and modified.
+
+In Pull Request, we try not to discuss the implementation of the code. The general implementation of the code and its logic should be determined in Issue. In the Pull Request, we only focus on the code format and code specification, so as to avoid wasting time caused by different opinions on implementation.
+
+## Specification
+
+### Pull Request Title
+
+Title Format: [`Pull Request Type`-`Issue No`][`Module Name`] `Pull Request Description`
+
+The corresponding relationship between `Pull Request Type` and `Issue Type` is as follows:
+
+<table>
+    <thead>
+        <tr>
+            <th style="width: 10%; text-align: center;">Issue Type</th>
+            <th style="width: 20%; text-align: center;">Pull Request Type</th>
+            <th style="width: 20%; text-align: center;">Example(Suppose Issue No is 3333)</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td style="text-align: center;">Feature</td>
+            <td style="text-align: center;">Feature</td>
+            <td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Bug</td>
+            <td style="text-align: center;">Fix</td>
+            <td style="text-align: center;">[Fix-3333][server] Fix xxx</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Improvement</td>
+            <td style="text-align: center;">Improvement</td>
+            <td style="text-align: center;">[Improvement-3333][alert] Improve the performance of xxx</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Test</td>
+            <td style="text-align: center;">Test</td>
+            <td style="text-align: center;">[Test-3333][api] Add the e2e test of xxx</td>
+        </tr>
+        <tr>
+            <td style="text-align: center;">Sub-Task</td>
+            <td style="text-align: center;">(Parent type corresponding to Sub-Task)</td>
+            <td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
+        </tr>
+    </tbody>
+</table>
+
+`Issue No` refers to the Issue number corresponding to the current Pull Request to be resolved, `Module Name` is the same as the `Module Name` of Issue.
+
+### Pull Request Branch
+
+Branch name format: `Pull Request type`-`Issue number`. e.g. Feature-3333
+
+### Pull Request Content
+
+Please refer to the commit message section.
+
+### Pull Request Code Style
+
+Code style is the thing you have to consider when you submit pull request for DolphinScheduler. We using [Checkstyle](https://checkstyle.sourceforge.io), a development tool to help programmers write Java code that adheres to a coding standard, in CI to keep DolphinScheduler codebase in the same style. Your pull request could not be merged if your code style checker failed. You could format your code by *Checkstyle* in your local environment before you submit your pull request to check code style. The activation step as below:
+
+1. Prepare Checkstyle configuration file: You could download it manually by [click here](https://github.com/apache/dolphinscheduler/blob/3.0.5/style/checkstyle.xml), but find it in DolphinScheduler repository would be a better way. You could find configuration file in the path `style/checkstyle.xml` after you clone repository from Github.
+
+2. Download Checkstyle plugins in Intellij IDEA: Search plugin by keyword **CheckStyle-IDEA** or install in [this page](https://plugins.jetbrains.com/plugin/1065-checkstyle-idea). You could see [install plugin](https://www.jetbrains.com/help/idea/managing-plugins.html#install_plugin_from_repo) if you do not know how to install plugin in Intellij IDEA
+
+3. Configure and activate Checkstyle and Intellij IDEA code-style: After completing the above steps, you could configure and activate it in your environment. You could find Checkstyle plugins in the path `Preferences -> Tool -> Checkstyle`. After that you could activate Checkstyles as screenshot show
+
+<p align="center">
+    <img src="../../../../img/contribute/join/pull-request/checkstyle-idea.png" alt="checkstyle idea configuration" />
+</p>
+
+For now your Checkstyle plugins are setup, it would show codes and files which out of style. We highly recommend you configure Intellij IDEA code-style for auto-formatting your code in Intellij IDEA, you could find this setting in `Preferences -> Editor -> Code Style -> Java` and then activate it as screenshot show
+
+<p align="center">
+    <img src="../../../../img/contribute/join/pull-request/code-style-idea.png" alt="code style idea configuration" />
+</p>
+
+1. Format your codebase in Intellij IDEA before submit your pull request: After you done above steps, you could using Intellij IDEA shortcut `Command + L`(for Mac) or `Ctrl+L`(for Windows) to format your code. The best time to format your code is before you commit your change to your local git repository.
+
+### Question
+
+- How to deal with one Pull Request to many Issues scenario.
+
+  First of all, there are fewer scenarios for one Pull Request to many Issues.
+  The root cause is that multiple issues need to do the same thing.
+  Usually, there are two solutions to this scenario: the first is to merge multiple issues with into the same issue, and then close the other issues;
+  the second is multiple issues have subtle differences.
+  In this scenario, the responsibilities of each issue can be clearly divided. The type of each issue is marked as Sub-Task, and then these sub task type issues are associated with one issue.
+  And each Pull Request is submitted should be associated with only one issue of a sub task.

docs/docs/en/contribute/join/review.md

@@ -0,0 +1,153 @@
+# Community Review
+
+Beside submit Issues and pull requests to the GitHub repository mentioned in [team](https://dolphinscheduler.apache.org/en-us/community), another important way to
+contribute to DolphinScheduler is reviewing GitHub Issues or Pull Requests. You can not only know the latest new and
+direction of the community, but also understand the good design in others during you reviewing. At the same time, you can
+increase your exposure in the community and accumulate your honor.
+
+Anyone is encouraged to review Issues and Pull Requests. We also raise a Help Wanted email discussion to solicit contributors
+from the community to review them. You could see detail in [mail][mail-review-wanted], we put the results of mail thread
+in [GitHub Discussion][discussion-result-review-wanted].
+
+> Note: It is only users mentioned in the [GitHub Discussion][discussion-result-review-wanted] can review Issues or Pull
+> Requests, Community advocates **Anyone is encouraged to review Issues and Pull Requests**. Users in 
+> [GitHub Discussion][discussion-result-review-wanted] show their willing to review when we collect in the mail thread.
+> The advantage of this list is when the community has discussion, in addition to the mention Members in [team](https://dolphinscheduler.apache.org/en-us/community),
+> you can also find some help in [GitHub Discussion][discussion-result-review-wanted] people. If you want to join the
+> [GitHub Discussion][discussion-result-review-wanted], please comment in that discussion and leave a module you are interested
+> in, and the maintainer will add you to the list.
+
+## How Reviewing
+
+DolphinScheduler receives community contributions through GitHub, and all its Issues and Pull Requests are hosted in GitHub.
+If you want to join community by reviewing, please go to section [review Issues](#issues), if you prefer Pull Requests please
+go to section [review Pull Requests](#pull-requests).
+
+### Issues
+
+Review Issues means discuss [Issues][all-issues] in GitHub and give suggestions on it. Include but are not limited to the following situations
+
+| Situation | Reason | Label | Action |
+| ------ | ------ | ------ | ------ |
+| wont fix | Has been fixed in dev branch | [wontfix][label-wontfix] | Close Issue, inform creator the fixed version if it already release |
+| duplicate issue | Had the same problem before | [duplicate][label-duplicate] | Close issue, inform creator the link of same issue |
+| Description not clearly | Without detail reproduce step | [need more information][label-need-more-information] | Inform creator add more description |
+
+In addition give suggestion, add label for issue is also important during review. The labeled issues can be retrieved
+better, which convenient for further processing. An issue can with more than one label. Common issue categories are:
+
+| Label | Meaning |
+| ------ | ------ |
+| [UI][label-UI] | UI and front-end related |
+| [security][label-security] | Security Issue |
+| [user experience][label-user-experience] | User experience Issue |
+| [development][label-development] | Development Issue |
+| [Python][label-Python] | Python Issue |
+| [plug-in][label-plug-in] | Plug-in Issue |
+| [document][label-document] | Document Issue |
+| [docker][label-docker] | Docker Issue |
+| [need verify][label-need-verify] | Need verify Issue |
+| [e2e][label-e2e] | E2E Issue |
+| [win-os][label-win-os] | windows operating system Issue |
+| [suggestion][label-suggestion] | Give suggestion to us |
+ 
+Beside classification, label could also set the priority of Issues. The higher the priority, the more attention pay
+in the community, the easier it is to be fixed or implemented. The priority label are as follows
+
+| Label | priority |
+| ------ | ------ |
+| [priority:high][label-priority-high] | High priority |
+| [priority:middle][label-priority-middle] | Middle priority |
+| [priority:low][label-priority-low] | Low priority |
+
+All the labels above in common label. For all labels in this project you could see in [full label list][label-all-list]
+
+Before reading following content, please make sure you have labeled the Issue.
+  
+* Remove label [Waiting for reply][label-waiting-for-reply] after replying: Label [Waiting for reply][label-waiting-for-reply]
+  added when [creating an Issue][issue-choose]. It makes positioning un reply issue more convenient, and you should remove
+  this label after you reviewed it. If you do not remove it, will cause others to waste time looking on the same issue.
+* Mark [Waiting for review][label-waiting-for-review] when not sure whether issue is resolved or not: There are two situations
+  when you review issue. One is the problem has been located or resolved, maybe have to [Create PR](./submit-code.md)
+  when necessary. Secondly, you are not sure about this issue, you can labeled [Waiting for review][label-waiting-for-review]
+  and mention others to make a second confirmation.
+
+When an Issue need to create Pull Requests, you could also labeled it from below.
+
+| Label | Mean |
+| ------ | ------ |
+| [Chore][label-Chore] | Chore for project |
+| [Good first issue][label-good-first-issue] | Good first issue for new contributor |
+| [easy to fix][label-easy-to-fix] | Easy to fix, harder than `Good first issue` |
+| [help wanted][label-help-wanted] | Help wanted |
+
+> Note: Only members have permission to add or delete label. When you need to add or remove lebals but are not member,
+> you can `@`  members to do that. But as long as you have a GitHub account, you can comment on issues and give suggestions.
+> We encourage everyone in the community to comment and answer issues
+
+### Pull Requests
+
+<!-- markdown-link-check-disable -->
+Review Pull mean discussing in [Pull Requests][all-PRs] in GitHub and giving suggestions to it. DolphinScheduler's 
+Pull Requests reviewing are the same as [GitHub's reviewing changes in pull requests][gh-review-pr]. You can give your
+suggestions in Pull Requests
+
+* When you think the Pull Request is OK to be merged, you can agree to the Pull Request according to the "Approve" process
+  in [GitHub's reviewing changes in pull requests][gh-review-pr].
+* When you think Pull Request needs to be changed, you can comment it according to the "Comment" process in 
+  [GitHub's reviewing changes in pull requests][gh-review-pr]. And when you think issues that must be fixed before they
+  merged, please follow "Request changes" in [GitHub's reviewing changes in pull requests][gh-review-pr] to ask contributors
+  modify it.
+<!-- markdown-link-check-enable -->
+
+Labeled Pull Requests is an important part. Reasonable classification can save a lot of time for reviewers. The good news
+is that the label's name and usage of Pull Requests are the same in [Issues](#issues), which can reduce the memory. For
+example, if there is a Pull Request is related to docker and block deployment. We can label it with [docker][label-docker]
+and [priority:high][label-priority-high].
+
+Pull Requests have some unique labels of it own
+
+| Label | Mean |
+| ------ | ------ |
+| [miss document][label-miss-document] | Pull Requests miss document, and should be add |
+| [first time contributor][label-first-time-contributor] | Pull Requests submit by first time contributor |
+| [don't merge][label-do-not-merge] | Pull Requests have some problem and should not be merged |
+
+> Note: Only members have permission to add or delete label. When you need to add or remove lebals but are not member,
+> you can `@`  members to do that. But as long as you have a GitHub account, you can comment on Pull Requests and give suggestions.
+> We encourage everyone in the community to review Pull Requests
+
+[mail-review-wanted]: https://lists.apache.org/thread/9flwlzrp69xjn6v8tdkbytq8glqp2k51
+[discussion-result-review-wanted]: https://github.com/apache/dolphinscheduler/discussions/7545
+[label-wontfix]: https://github.com/apache/dolphinscheduler/labels/wontfix
+[label-duplicate]: https://github.com/apache/dolphinscheduler/labels/duplicate
+[label-need-more-information]: https://github.com/apache/dolphinscheduler/labels/need%20more%20information
+[label-win-os]: https://github.com/apache/dolphinscheduler/labels/win-os
+[label-waiting-for-reply]: https://github.com/apache/dolphinscheduler/labels/Waiting%20for%20reply
+[label-waiting-for-review]: https://github.com/apache/dolphinscheduler/labels/Waiting%20for%20review
+[label-user-experience]: https://github.com/apache/dolphinscheduler/labels/user%20experience
+[label-development]: https://github.com/apache/dolphinscheduler/labels/development
+[label-UI]: https://github.com/apache/dolphinscheduler/labels/UI
+[label-suggestion]: https://github.com/apache/dolphinscheduler/labels/suggestion
+[label-security]: https://github.com/apache/dolphinscheduler/labels/security
+[label-Python]: https://github.com/apache/dolphinscheduler/labels/Python
+[label-plug-in]: https://github.com/apache/dolphinscheduler/labels/plug-in
+[label-document]: https://github.com/apache/dolphinscheduler/labels/document
+[label-docker]: https://github.com/apache/dolphinscheduler/labels/docker
+[label-all-list]: https://github.com/apache/dolphinscheduler/labels
+[label-Chore]: https://github.com/apache/dolphinscheduler/labels/Chore
+[label-good-first-issue]: https://github.com/apache/dolphinscheduler/labels/good%20first%20issue
+[label-help-wanted]: https://github.com/apache/dolphinscheduler/labels/help%20wanted
+[label-easy-to-fix]: https://github.com/apache/dolphinscheduler/labels/easy%20to%20fix
+[label-priority-high]: https://github.com/apache/dolphinscheduler/labels/priority%3Ahigh
+[label-priority-middle]: https://github.com/apache/dolphinscheduler/labels/priority%3Amiddle
+[label-priority-low]: https://github.com/apache/dolphinscheduler/labels/priority%3Alow
+[label-miss-document]: https://github.com/apache/dolphinscheduler/labels/miss%20document
+[label-first-time-contributor]: https://github.com/apache/dolphinscheduler/labels/first%20time%20contributor
+[label-do-not-merge]: https://github.com/apache/dolphinscheduler/labels/don%27t%20merge
+[label-e2e]: https://github.com/apache/dolphinscheduler/labels/e2e
+[label-need-verify]: https://github.com/apache/dolphinscheduler/labels/need%20to%20verify
+[issue-choose]: https://github.com/apache/dolphinscheduler/issues/new/choose
+[all-issues]: https://github.com/apache/dolphinscheduler/issues
+[all-PRs]: https://github.com/apache/dolphinscheduler/pulls
+[gh-review-pr]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews

docs/docs/en/contribute/join/security.md

@@ -0,0 +1,8 @@
+# Security
+
+The Apache Software Foundation takes a rigorous stance on eliminating security issues in its software projects. Apache DolphinScheduler is also very concerned Security issues related to its features and functionality.
+
+If you have apprehensions regarding DolphinScheduler’s security or you discover vulnerability or potential threat, don’t hesitate to get in touch with the Apache Security Team by dropping a mail at [security@apache.org](mailto:security@apache.org). Please specify the project name as DolphinScheduler in the email and provide a description of the relevant problem or potential threat. You are also urged to recommend the way to reproduce and replicate the issue. The apache security team and the DolphinScheduler community will get back to you after assessing and analysing the findings.
+
+Please pay attention to report the security issue on the security email before disclosing it on public domain.
+

docs/docs/en/contribute/join/submit-code.md

@@ -0,0 +1,63 @@
+# Submit Code
+
+* First from the remote repository *https://github.com/apache/dolphinscheduler.git* fork a copy of the code into your own repository
+
+* There are currently three branches in the remote repository:
+    * master           normal delivery branch
+        After the stable release, merge the code from the stable branch into the master.
+    
+    * dev              daily development branch
+        Every day dev development branch, newly submitted code can pull request to this branch.
+
+
+* Clone your repository to your local
+    `git clone https://github.com/apache/dolphinscheduler.git`
+
+* Add remote repository address, named upstream
+    `git remote add upstream https://github.com/apache/dolphinscheduler.git`
+
+* View repository
+    `git remote -v`
+
+>At this time, there will be two repositories: origin (your own repository) and upstream (remote repository)
+
+* Get/Update remote repository code
+    `git fetch upstream`
+
+* Synchronize remote repository code to local repository
+
+```
+git checkout origin/dev
+git merge --no-ff upstream/dev
+```
+
+If remote branch has a new branch such as `dev-1.0`, you need to synchronize this branch to the local repository
+      
+```
+git checkout -b dev-1.0 upstream/dev-1.0
+git push --set-upstream origin dev-1.0
+```
+
+* Create new branch
+```
+git checkout -b xxx origin/dev
+```
+
+Make sure that the branch `xxx` is building successfully on the latest code of the official dev branch
+* After modifying the code locally in the new branch, submit it to your own repository:
+  
+`git commit -m 'commit content'`
+    
+`git push origin xxx --set-upstream`
+
+* Submit changes to the remote repository
+
+* On the github page, click "New pull request".
+
+* Select the modified local branch and the branch you want to merge with the past, click "Create pull request".
+
+* Then the community Committers will do CodeReview, and then he will discuss some details (including design, implementation, performance, etc.) with you. When everyone on the team is satisfied with this modification, the commit will be merged into the dev branch
+
+* Finally, congratulations, you have become an official contributor to dolphinscheduler!
+
+

docs/docs/en/contribute/join/subscribe.md

@@ -0,0 +1,23 @@
+# Subscribe Mailing Lists
+
+It is highly recommended to subscribe to the development mailing list to keep up-to-date with the community.
+
+In the process of using DolphinScheduler, if you have any questions or ideas, suggestions, you can participate in the DolphinScheduler community building through the Apache mailing list. Sending a subscription email is also very simple, the steps are as follows:
+
+1. Send an email to dev-subscribe@dolphinscheduler.apache.org with your own email address, subject and content are arbitrary.
+
+2. Receive confirmation email and reply. After completing step 1, you will receive a confirmation email from dev-help@dolphinscheduler.apache.org (if not received, please confirm whether the email is automatically classified as spam, promotion email, subscription email, etc.) . Then reply directly to the email, or click on the link in the email to reply quickly, the subject and content are arbitrary.
+
+3. Receive a welcome email. After completing the above steps, you will receive a welcome email with the subject WELCOME to dev@dolphinscheduler.apache.org, and you have successfully subscribed to the Apache DolphinScheduler mailing list.
+
+# Unsubscribe Mailing Lists
+
+If you do not need to know what's going on with DolphinScheduler, you can unsubscribe from the mailing list.
+
+Unsubscribe from the mailing list steps are as follows:
+
+1. Send an email to dev-unsubscribe@dolphinscheduler.apache.org with your subscribed email address, subject and content are arbitrary.
+
+2. Receive confirmation email and reply. After completing step 1, you will receive a confirmation email from dev-help@dolphinscheduler.apache.org (if not received, please confirm whether the email is automatically classified as spam, promotion email, subscription email, etc.) . Then reply directly to the email, or click on the link in the email to reply quickly, the subject and content are arbitrary.
+
+3. Receive a goodbye email. After completing the above steps, you will receive a goodbye email with the subject GOODBYE from dev@dolphinscheduler.apache.org, and you have successfully unsubscribed to the Apache DolphinScheduler mailing list, and you will not receive emails from dev@dolphinscheduler.apache.org.

docs/docs/en/contribute/join/unit-test.md

@@ -0,0 +1,118 @@
+## Unit Test Coverage
+
+### 1. The Benefits of Writing Unit Tests
+
+-    Unit tests help everyone to get into the details of the code and understand how it works.
+-    Through test cases we can find bugs and submit robust code.
+-    The test case is also a demo usage of the code.
+
+### 2. Some design principles for unit test cases
+
+-    The steps, granularity and combination of conditions should be carefully designed.
+-    Pay attention to boundary conditions.
+-    Unit tests should be well designed as well as avoiding useless code.
+-    When you find a `method` is difficult to write unit test, and if you confirm that the `method` is `bad code`, then refactor it with the developer.
+<!-- markdown-link-check-disable -->
+-    DolphinScheduler: [mockito](http://site.mockito.org/). Here are some development guides: [mockito tutorial](http://www.baeldung.com/bdd-mockito), [mockito refcard](https://dzone.com/refcardz/mockito)
+<!-- markdown-link-check-enable -->
+-    TDD(option): When you start writing a new feature, you can try writing test cases first.
+
+### 3. Test coverage setpoint
+
+-    At this stage, the default value for test coverage of Delta change codes is >= 60%, the higher the better.
+-    We can see the test reports on this page:  https://codecov.io/gh/apache/dolphinscheduler
+
+## Fundamental guidelines for unit test
+
+### 1. Isolation and singleness
+
+A test case should be accurate to the method level, and it should be possible to execute the test case alone. At the same time the focus is always on the method (only the method is tested).
+
+If the method is too complex, it should be split up again during the development phase. For test cases, it is best that a case focuses on only one branch (judgment). When changes are applied to it, they only affect the success of a test case. This will greatly facilitate our verification of issues and problem solving during the development phase. At the same time, however, it also poses a great challenge in terms of coverage.
+
+### 2. Automaticity
+
+Unit tests can be automated. Mandatory: all unit tests must be written under src/test. Also the method naming should conform to the specification. Benchmark tests are excluded.
+
+### 3. reproducibility
+
+Multiple executions (any environment, any time) result in unique and repeatable results.
+
+### 4. Lightweight
+
+That is, any environment can be implemented quickly.
+
+This requires that we don't rely on too many components, such as various spring beans and the like. These are all mock in unit tests, nd adding them would increase the speed of our single-test execution, as well as potentially passing on contamination.
+
+For some databases, other external components, etc. As far as possible, the mock client is not dependent on the external environment (the presence of any external dependencies greatly limits the portability and stability of test cases and the correctness of results), which also makes it easy for developers to test in any environment.
+
+### 5. Measurable
+
+Over the years, mockito has grown to be the NO.1 mock, but it still doesn't support mock static methods, constructors, etc. Even the website keeps saying: "Don't mock everything". So use static methods as little as possible.
+
+It is generally recommended to provide static methods only in some utility classes, in which case you don't need mocks and just use real classes. If the dependent class is not a utility class, static methods can be refactored into instance methods. This is more in line with the object-oriented design concept.
+
+### 6. Completeness
+
+Test coverage, this is a very difficult problem. For the core process, we hope to achieve 90% coverage, non-core process requirements more than 60%.
+
+High enough coverage will reduce the probability of bugs and also reduce the cost of our regression tests. This is a long process, and whenever developers add or modify code, test cases need to be refined at the same time. We hope developers and relevant code reviewer will pay enough attention to this point.
+
+### 7. Refusion invalid assertion
+
+Invalid assertions make the test itself meaningless, it has little to do with whether your code is correct or not. And there is a risk of creating an illusion of success that may last until your code is deploying to production.
+
+There are several types of invalid assertions:
+
+1.   Different types of comparisons.
+
+2.   Determines that an object or variable with a default value is not null.
+
+     This seems meaningless. Therefore, when making the relevant judgements you should pay attention to whether it contains a default value itself.
+
+3.   Assertions should be affirmative rather than negative if possible. Assertions should be within a range of predicted results, or exact values, whenever possible (otherwise you may end up with something that doesn't match your actual expectations but passes the assertion) unless your code only cares about whether it is empty or not.
+
+### 8. Some points to note for unit tests
+
+1: Thread.sleep()
+
+Try not to use Thread.sleep in your test code, it makes the test unstable and may fail unexpectedly due to the environment or load. The following approach is recommended.
+
+`Awaitility.await().atMost(...)`
+
+2: Ignore some test classes
+
+The @Ignore annotation should be linked to the relevant issue address so that subsequent developers can track the history of why the test was ignored.
+
+For example @Ignore("see #1").
+
+3: try-catch Unit test exception
+
+The test will fail when the code in the unit test throws an exception. Therefore, there is no need to use try-catch to catch exceptions.
+
+     ```java
+     @Test
+     public void testMethod() {
+       try {
+                 // Some code
+       } catch (MyException e) {
+         Assert.fail(e.getMessage());  // Noncompliant
+       }
+     }
+     ```
+You should this: 
+
+```java
+@Test
+public void testMethod() throws MyException {
+    // Some code
+}
+```
+
+4: Test exceptions
+
+When you need to test for exceptions, you should avoid including multiple method invocations in your test code (especially if there are multiple methods that can raise the same exception), and you should clearly state what you are testing for.
+
+5: Refuse to use MockitoJUnitRunner.Silent.class
+
+When an UnnecessaryStubbingException occurs in a unit test, do not first consider using @RunWith(MockitoJUnitRunner.Silent.class) to resolve it. This just hides the problem, and you should follow the exception hint to resolve the issue in question, which is not a difficult task. When the changes are done, you will find that your code is much cleaner again.

docs/docs/en/contribute/release/release-post.md

@@ -0,0 +1,31 @@
+# Release Post
+
+We still have some publish task to do after we send the announcement mail, currently we have to publish Docker images to
+Docker Hub and also publish pydolphinscheduler to PyPI.
+
+## Publish Docker Image
+
+we already have the exists CI to publish the latest Docker image to GitHub container register with [config](https://github.com/apache/dolphinscheduler/blob/d80cf21456265c9d84e642bdb4db4067c7577fc6/.github/workflows/publish-docker.yaml#L55-L63).
+We could reuse the main command the CI run and publish our Docker images to Docker Hub by single command.
+
+```bash
+# Please change the <VERSION> place hold to the version you release
+./mvnw -B clean deploy \
+    -Dmaven.test.skip \
+    -Dmaven.javadoc.skip \
+    -Dmaven.checkstyle.skip \
+    -Dmaven.deploy.skip \
+    -Ddocker.tag=<VERSION> \
+    -Ddocker.hub=apache \
+    -Pdocker,release
+```
+
+## Publish pydolphinscheduler to PyPI
+
+Python API need to release to PyPI for easier download and use, you can see more detail in [Python API release](https://github.com/apache/dolphinscheduler-sdk-python/blob/main/RELEASE.md#to-pypi)
+to finish PyPI release.
+
+## Get All Contributors
+
+You might need all contributors in current release when you want to publish the release news or announcement, you could
+use command `python release.py contributor` in directory `tools/release` to auto generate contributor GitHub id.

docs/docs/en/contribute/release/release-prepare.md

@@ -0,0 +1,31 @@
+# Release Preparation
+
+## Check release-docs
+
+Compared with the last release, the `release-docs` of the current release needs to be updated to the latest, if there are dependencies and versions changes
+
+ - `dolphinscheduler-dist/release-docs/LICENSE`
+ - `dolphinscheduler-dist/release-docs/NOTICE`
+ - `dolphinscheduler-dist/release-docs/licenses`
+
+## Update Version
+
+For example, to release `x.y.z`, the following updates are required:
+
+- Version in the code:
+  - `sql`:
+    - `dolphinscheduler_mysql.sql`: `t_ds_version` needs to be updated to x.y.z
+    - `dolphinscheduler_postgre.sql`: `t_ds_version` needs to be updated to x.y.z
+    - `dolphinscheduler_h2.sql`: `t_ds_version` needs to be updated to x.y.z
+    - `upgrade`: whether to add`x.y.z_schema`
+    - `soft_version`: need to be updated to x.y.z
+  - `deploy/docker/.env`: `HUB` change to `apache`，`TAG` change to `x.y.z`
+  - `deploy/kubernetes/dolphinscheduler`:
+    - `Chart.yaml`: `appVersion` needs to be updated to x.y.z (`version` is helm chart version，incremented and different from x.y.z)
+    - `values.yaml`: `image.tag` needs to be updated to x.y.z
+  - `dolphinscheduler-python/pydolphinscheduler/setup.py`: change `version` to x.y.z
+- Version in the docs:
+  - Change the placeholder `<version>`(except `pom`)  to the `x.y.z` in directory `docs`
+  - Add new history version
+    - `docs/docs/en/history-versions.md` and `docs/docs/zh/history-versions.md`: Add the new version and link for `x.y.z`
+  - `docs/configs/docsdev.js`: change `/dev/` to `/x.y.z/`, **DO NOT** change this filename, is will be auto change by website tools.

docs/docs/en/contribute/release/release.md

@@ -0,0 +1,542 @@
+# Release Guide
+
+## Check Your Environment
+
+To make sure you could successfully complete the release for DolphinScheduler, you should check your environment and make sure
+all conditions are met, if any or them are missing, you should install them and make sure them work.
+
+```shell
+# JDK 1.8 above is requests
+java -version
+# Maven requests
+mvn -version
+# Python 3.6 above is requests, and you have to make keyword `python` work in your terminal and version match
+python --version
+```
+
+## GPG Settings
+
+### Install GPG
+
+Download installation package on [official GnuPG website](https://www.gnupg.org/download/index.html).
+The command of GnuPG 1.x version can differ a little from that of 2.x version.
+The following instructions take `GnuPG-2.1.23` version for example.
+
+After the installation, execute the following command to check the version number.
+
+```shell
+gpg --version
+```
+
+### Create Key
+
+After the installation, execute the following command to create key.
+
+This command indicates `GnuPG-2.x` can be used:
+
+```shell
+gpg --full-gen-key
+```
+
+This command indicates `GnuPG-1.x` can be used:
+
+```shell
+gpg --gen-key
+```
+
+Finish the key creation according to instructions, **Notice: Please use Apache mails and its password for key creation.**
+
+```shell
+gpg (GnuPG) 2.0.12; Copyright (C) 2009 Free Software Foundation, Inc.
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
+
+Please select what kind of key you want:
+  (1) RSA and RSA (default)
+  (2) DSA and Elgamal
+  (3) DSA (sign only)
+  (4) RSA (sign only)
+Your selection? 1
+RSA keys may be between 1024 and 4096 bits long.
+What keysize do you want? (2048) 4096
+Requested keysize is 4096 bits
+Please specify how long the key should be valid.
+        0 = key does not expire
+     <n>  = key expires in n days
+     <n>w = key expires in n weeks
+     <n>m = key expires in n months
+     <n>y = key expires in n years
+Key is valid for? (0)
+Key does not expire at all
+Is this correct? (y/N) y
+
+GnuPG needs to construct a user ID to identify your key.
+
+Real name: ${Input username}
+Email address: ${Input email}
+Comment: ${Input comment}
+You selected this USER-ID:
+   "${Inputed username} (${Inputed comment}) <${Inputed email}>"
+
+Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? O
+You need a Passphrase to protect your secret key. # Input your Apache mail passwords
+```
+
+### Check Generated Key
+
+```shell
+gpg --list-keys
+```
+
+Execution Result:
+
+```shell
+pub   4096R/85E11560 2019-11-15
+uid                  ${Username} (${Comment}) <{Email}>
+sub   4096R/A63BC462 2019-11-15
+```
+
+Among them, 85E11560 is public key ID.
+
+### Upload the Public Key to Key Server
+
+The command is as follow:
+
+```shell
+gpg --keyserver hkp://pool.sks-keyservers.net --send-key 85E11560
+```
+
+`pool.sks-keyservers.net` is randomly chosen from [public key server](https://sks-keyservers.net/status/).
+Each server will automatically synchronize with one another, so it would be okay to choose any one, a backup keys servers
+is `gpg --keyserver hkp://keyserver.ubuntu.com --send-key <YOUR_KEY_ID>`
+
+## Apache Maven Central Repository Release
+
+### Set `settings-security.xml` and `settings.xml`
+
+In this section, we add Apache server maven configuration to prepare the release, we have to add `settings-security.xml` according
+to [here](http://maven.apache.org/guides/mini/guide-encryption.html) firstly and then change your `~/.m2/settings.xml` like below
+
+```xml
+<settings>
+  <servers>
+    <server>
+      <id>apache.snapshots.https</id>
+      <username> <!-- APACHE LDAP username --> </username>
+      <password> <!-- APACHE LDAP encrypted password --> </password>
+    </server>
+    <server>
+      <id>apache.releases.https</id>
+      <username> <!-- APACHE LDAP username --> </username>
+      <password> <!-- APACHE LDAP encrypted password --> </password>
+    </server>
+  </servers>
+</settings>
+```
+
+### Set Release in Environment
+
+We will use the release version, your github name and your Apache username below several times, so it is better to store
+it to bash variable for easier use.
+
+```shell
+VERSION=<THE-VERSION-YOU-RELEASE>
+GH_USERNAME=<YOUR-GITHUB-USERNAME>
+A_USERNAME=<YOUR-APACHE-USERNAME>
+```
+
+> Note: We can use the variable directly in you bash after we set environment, without changing anything. For example, we
+> can use command `git clone -b "${VERSION}"-prepare https://github.com/apache/dolphinscheduler.git` to clone the release branch
+> and it can be success by covert the `"${VERSION}"` to `<THE-VERSION-YOU-RELEASE>`. But you have to change `<VERSION>` manually in
+> some of not bash step like [vote mail](#vote-procedure), we using `<VERSION>` instead of `"${VERSION}"` to notice release
+> manager they have to change by hand.
+
+### Create Release Branch
+
+In this section, we dwonload source code from github and create new branch to release
+
+```shell
+git clone -b "${VERSION}"-prepare https://github.com/apache/dolphinscheduler.git
+cd ~/dolphinscheduler/
+git pull
+git checkout -b "${VERSION}"-release
+git push origin "${VERSION}"-release
+```
+
+### Pre-Release Check
+
+```shell
+# make gpg command could be run in maven correct
+export GPG_TTY=$(tty)
+
+mvn release:prepare -Prelease,python -Darguments="-Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true" -DautoVersionSubmodules=true -DdryRun=true -Dusername="${GH_USERNAME}"
+```
+
+* `-Prelease,python`: choose release and python profile, which will pack all the source codes, jar files and executable binary packages, and Python distribute package.
+* `-DautoVersionSubmodules=true`: it can make the version number is inputted only once and not for each sub-module.
+* `-DdryRun=true`: dry run which means not to generate or submit new version number and new tag.
+
+### Prepare for the Release
+
+First, clean local pre-release check information.
+
+```shell
+mvn release:clean
+```
+
+Then, prepare to execute the release.
+
+```shell
+mvn release:prepare -Prelease,python -Darguments="-Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true" -DautoVersionSubmodules=true -DpushChanges=false -Dusername="${GH_USERNAME}"
+```
+
+It is basically the same as the previous rehearsal command, but deleting `-DdryRun=true` parameter.
+
+* `-DpushChanges=false`: do not submit the edited version number and tag to GitHub automatically.
+
+> Note: You have to config your git `user.name` and `user.password` by command `git config --global user.email "you@example.com"`
+> and `git config --global user.name "Your Name"` if you meet some mistake like **Please tell me who you are.**
+> from git.
+
+After making sure there is no mistake in local files, submit them to GitHub.
+
+```shell
+git push -u origin "${VERSION}"-release
+git push origin --tags
+```
+
+<!-- markdown-link-check-disable -->
+
+> Note1: In this step, you should use github token for password because native password no longer supported, you can see
+> https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token for more
+> detail about how to create token about it.
+
+> Note2: After the command done, it will auto-created `release.properties` file and `*.Backup` files, their will be need
+> in the following command and DO NOT DELETE THEM
+
+<!-- markdown-link-check-enable -->
+
+### Deploy the Release
+
+```shell
+mvn release:perform -Prelease,python -Darguments="-Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true" -DautoVersionSubmodules=true -Dusername="${GH_USERNAME}"
+```
+
+After that command is executed, the version to be released will be uploaded to Apache staging repository automatically.
+Go to [apache staging repositories](https://repository.apache.org/#stagingRepositories) and login by Apache LDAP. then you can see the uploaded version, the content of `Repository` column is the `${STAGING.REPOSITORY}`.
+Click `Close` to tell Nexus that the construction is finished, because only in this way, this version can be usable.
+If there is any problem in gpg signature, `Close` will fail, but you can see the failure information through `Activity`.
+
+## Apache SVN Repository Release
+
+### Checkout dolphinscheduler Release Directory
+
+If there is no local work directory, create one at first.
+
+```shell
+mkdir -p ~/ds_svn/dev/
+cd ~/ds_svn/dev/
+```
+
+After the creation, checkout dolphinscheduler release directory from Apache SVN.
+
+```shell
+svn --username="${A_USERNAME}" co https://dist.apache.org/repos/dist/dev/dolphinscheduler
+cd ~/ds_svn/dev/dolphinscheduler
+```
+
+### Export your new gpg KEYS to release(optional)
+
+Only if the first time you release with this gpg KEY, including it is you first release, or you change your KEY. You should
+change working directory to another one because this step need checkout and change KEYS in release directory.
+
+```shell
+mkdir -p ~/ds_svn/release/
+cd ~/ds_svn/release/
+svn --username="${A_USERNAME}" co https://dist.apache.org/repos/dist/release/dolphinscheduler
+gpg -a --export <YOUR-GPG-KEY-ID> >> KEYS
+svn add *
+svn --username="${A_USERNAME}" commit -m "new key <YOUR-GPG-KEY-ID> add"
+```
+
+> NOTE: it may take a few minutes to sync to the mirror in your first time checkout, because it will download all the files
+
+### Add the Release Content to SVN Directory
+
+Create folder by version number.
+
+```shell
+mkdir -p ~/ds_svn/dev/dolphinscheduler/"${VERSION}"
+mkdir -p ~/ds_svn/dev/dolphinscheduler/"${VERSION}"/python
+cd ~/ds_svn/dev/dolphinscheduler/"${VERSION}"
+```
+
+Add source code packages, binary packages and executable binary packages to SVN working directory.
+
+```shell
+# Source and binary tarball for main code
+cp -f ~/dolphinscheduler/dolphinscheduler-dist/target/*.tar.gz ~/ds_svn/dev/dolphinscheduler/"${VERSION}"
+cp -f ~/dolphinscheduler/dolphinscheduler-dist/target/*.tar.gz.asc ~/ds_svn/dev/dolphinscheduler/"${VERSION}"
+
+# Source and binary tarball for Python API
+cp -f ~/dolphinscheduler/dolphinscheduler-dist/target/python/* ~/ds_svn/dev/dolphinscheduler/"${VERSION}"/python
+```
+
+### Generate sign files
+
+```shell
+shasum -a 512 apache-dolphinscheduler-"${VERSION}"-src.tar.gz >> apache-dolphinscheduler-"${VERSION}"-src.tar.gz.sha512
+shasum -b -a 512 apache-dolphinscheduler-"${VERSION}"-bin.tar.gz >> apache-dolphinscheduler-"${VERSION}"-bin.tar.gz.sha512
+cd python
+shasum -a 512 apache-dolphinscheduler-python-"${VERSION}".tar.gz >> apache-dolphinscheduler-python-"${VERSION}".tar.gz.sha512
+shasum -b -a 512 apache_dolphinscheduler-python-"${VERSION}"-py3-none-any.whl >> apache_dolphinscheduler-python-"${VERSION}"-py3-none-any.whl.sha512
+cd ../
+```
+
+### Commit to Apache SVN
+
+```shell
+cd ~/ds_svn/dev/dolphinscheduler
+svn add *
+svn --username="${A_USERNAME}" commit -m "release ${VERSION}"
+```
+## Check Release
+
+### Check sha512 hash
+
+```shell
+shasum -c apache-dolphinscheduler-"${VERSION}"-src.tar.gz.sha512
+shasum -c apache-dolphinscheduler-"${VERSION}"-bin.tar.gz.sha512
+cd python
+shasum -c apache-dolphinscheduler-python-"${VERSION}".tar.gz.sha512
+shasum -c apache_dolphinscheduler-python-"${VERSION}"-py3-none-any.whl.sha512
+cd ../
+```
+
+### Check gpg Signature
+
+First, import releaser's public key.
+Import KEYS from SVN repository to local. (The releaser does not need to import again; the checking assistant needs to import it, with the user name filled as the releaser's. )
+
+```shell
+curl https://dist.apache.org/repos/dist/release/dolphinscheduler/KEYS >> KEYS
+gpg --import KEYS
+gpg --edit-key "${A_USERNAME}"
+  > trust
+
+Please decide how far you trust this user to correctly verify other users' keys
+(by looking at passports, checking fingerprints from different sources, etc.)
+
+  1 = I don't know or won't say
+  2 = I do NOT trust
+  3 = I trust marginally
+  4 = I trust fully
+  5 = I trust ultimately
+  m = back to the main menu
+
+Your decision? 5
+
+  > save
+```
+
+Then, check the gpg signature.
+
+```shell
+gpg --verify apache-dolphinscheduler-"${VERSION}"-src.tar.gz.asc
+gpg --verify apache-dolphinscheduler-"${VERSION}"-bin.tar.gz.asc
+cd python
+gpg --verify apache-dolphinscheduler-python-"${VERSION}".tar.gz.asc
+gpg --verify apache_dolphinscheduler-python-"${VERSION}"-py3-none-any.whl.asc
+cd ../
+```
+
+> Note: You have to create gpg signature manually when you can not find your `asc` file, the command
+> `gpg --armor --detach-sign --digest-algo=SHA512 apache-dolphinscheduler-"${VERSION}"-bin.tar.gz` and
+> `gpg --armor --detach-sign --digest-algo=SHA512 apache-dolphinscheduler-"${VERSION}"-src.tar.gz` will create them
+
+### Check Released Files
+
+#### Check source package
+
+Decompress `apache-dolphinscheduler-<VERSION>-src.tar.gz` and `python/apache-dolphinscheduler-python-<VERSION>.tar.gz` then check the following items:
+
+*   Check whether source tarball is oversized for including nonessential files
+*   `LICENSE` and `NOTICE` files exist
+*   Correct year in `NOTICE` file
+*   There is only text files but no binary files
+*   All source files have ASF headers
+*   Codes can be compiled and pass the unit tests (mvn install)
+*   The contents of the release match with what's tagged in version control (diff -r a verify_dir tag_dir)
+*   Check if there is any extra files or folders, empty folders for example
+
+#### Check binary packages
+
+Decompress `apache-dolphinscheduler-<VERSION>-src.tar.gz` and `python/apache-dolphinscheduler-python-<VERSION>-bin.tar.gz`
+to check the following items:
+
+- `LICENSE` and `NOTICE` files exist
+- Correct year in `NOTICE` file
+- Check the third party dependency license:
+  - The software have a compatible license
+  - All software licenses mentioned in `LICENSE`
+  - All the third party dependency licenses are under `licenses` folder
+  - If it depends on Apache license and has a `NOTICE` file, that `NOTICE` file need to be added to `NOTICE` file of the release
+
+## Call for a Vote
+
+### Update Release Notes
+
+You should create a release note in GitHub by [new release note](https://github.com/apache/dolphinscheduler/releases/new).
+It should be done before vote mail because we need the release note in the mail. You could use command
+`python release.py changelog` in directory `tools/release` to creat the changelog.
+
+> NOTE: Or if you prefer to create manually, you can use command `git log --pretty="- %s" <PREVIOUS-RELEASE-SHA>..<CURRENT-RELEASE-SHA> > changelog.md`
+> (some log maybe not correct, you should filter them by yourself) and classify them and paste them to GitHub release note page
+
+### Vote procedure
+
+1. DolphinScheduler community vote: send the vote e-mail to `dev@dolphinscheduler.apache.org`.
+PMC needs to check the rightness of the version according to the document before they vote.
+After at least 72 hours and with at least 3 `+1 and no -1 PMC member` votes, it can come to the next stage of the vote.
+
+2. Announce the vote result: send the result vote e-mail to `dev@dolphinscheduler.apache.org`。
+
+### Vote Templates
+
+#### DolphinScheduler Community Vote Template
+
+Title：
+
+```txt
+[VOTE] Release Apache DolphinScheduler <VERSION>
+```
+
+Body：
+
+```txt
+Hello DolphinScheduler Community,
+
+This is a call for vote to release Apache DolphinScheduler version <VERSION>
+
+Release notes: https://github.com/apache/dolphinscheduler/releases/tag/<VERSION>
+
+The release candidates: https://dist.apache.org/repos/dist/dev/dolphinscheduler/<VERSION>/
+
+Maven 2 staging repository: https://repository.apache.org/content/repositories/<STAGING.REPOSITORY>/org/apache/dolphinscheduler/
+
+Git tag for the release: https://github.com/apache/dolphinscheduler/tree/<VERSION>
+
+Release Commit ID: https://github.com/apache/dolphinscheduler/commit/<SHA-VALUE>
+
+Keys to verify the Release Candidate: https://downloads.apache.org/dolphinscheduler/KEYS
+
+Look at here for how to verify this release candidate: https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/contribute/release/release.html
+
+The vote will be open for at least 72 hours or until necessary number of votes are reached.
+
+Please vote accordingly:
+
+[ ] +1 approve
+[ ] +0 no opinion
+[ ] -1 disapprove with the reason
+
+Checklist for reference:
+
+[ ] Download links are valid.
+[ ] Checksums and PGP signatures are valid.
+[ ] Source code artifacts have correct names matching the current release.
+[ ] LICENSE and NOTICE files are correct for each DolphinScheduler repo.
+[ ] All files have license headers if necessary.
+[ ] No compiled archives bundled in source archive.
+```
+
+2. Announce the vote result:
+
+Body：
+
+```txt
+The vote to release Apache DolphinScheduler <VERSION> has passed.Here is the vote result,
+
+4 PMC member +1 votes:
+
+xxx
+xxx
+xxx
+xxx
+
+1 community +1 vote:
+xxx
+
+Thanks everyone for taking time to check this release and help us.
+```
+
+## Finish the Release
+
+### Move source packages, binary packages from the `dev` directory to `release` directory
+
+```shell
+svn mv https://dist.apache.org/repos/dist/dev/dolphinscheduler/"${VERSION}" https://dist.apache.org/repos/dist/release/dolphinscheduler/
+```
+
+### Update Document
+
+Website should be present before you send the announce mail this section will tell you how to change the website. For example,
+the release version is `<VERSION>`, the following updates are required(note it will take effect immediately when the PR is merged):
+
+- Repository **apache/dolphinscheduler-website**:
+  - `download/en-us/download.md` and `download/zh-cn/download.md`: add the download of the `<VERSION>` release package
+  - `scripts/conf.sh`: Add new release version `<VERSION>` key-value pair to variable `DEV_RELEASE_DOCS_VERSIONS`
+- Repository **apache/dolphinscheduler**:
+  - `docs/configs/site.js`:
+    - `docsLatest`: update to `<VERSION>`
+    - `docs0`: The `text` of two places of `en-us/zh-cn` needs to be updated to `latest(<VERSION>)`
+    - `docsxyz`: Add a drop-down menu with `key` as `docsxyz` and `text` as `<VERSION>` in `children` of two places of `en-us/zh-cn`
+  - `docs/configs/index.md.jsx`: Add `<VERSION>: docsxyzConfig`
+  - `docs/docs/en/history-versions.md` and `docs/docs/zh/history-versions.md`: Add new `<VERSION>` release docs.
+  - `.github/ISSUE_TEMPLATE/bug-report.yml`: DolphinScheduler's GitHub [bug-report](https://github.com/apache/dolphinscheduler/blob/dev/.github/ISSUE_TEMPLATE/bug-report.yml)
+    issue template have **Version** selection bottom. So after we release DolphinScheduler we should and the new `<VERSION>` to
+    bug-report.yml
+
+### Publish Docker Image and Helm Chart
+
+We have a [workflow](../../../../.github/workflows/publish-docker.yaml) to automatically publish Docker images
+and a [workflow](../../../../.github/workflows/publish-helm-chart.yaml) to automatically publish Helm Chart to Docker Hub,
+after you change the release from "pre-release" to "release", the workflow will be triggered. All you need to do
+is to observe the aforementioned workflows, and after they are completed, you can pull the Docker images locally and
+verify that they work as expected.
+
+### Send Announcement E-mail Community
+
+You should send announcement E-mail after release process finished. The E-mail should send to `dev@dolphinscheduler.apache.org`
+and cc to `announce@apache.org`.
+
+Announcement e-mail template as below：
+
+Title：
+
+```txt
+[ANNOUNCE] Release Apache DolphinScheduler <VERSION>
+```
+
+Body：
+
+```txt
+Hi all,
+
+We are glad to announce the release of Apache DolphinScheduler <VERSION>. Once again I would like to express my thanks to your help.
+
+Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduler system,
+dedicated to solving the complex task dependencies in data processing, making the scheduler system out of the box for data processing.
+
+
+Download Links: https://dolphinscheduler.apache.org/en-us/download
+
+Release Notes: https://github.com/apache/dolphinscheduler/releases/tag/<VERSION>
+
+Website: https://dolphinscheduler.apache.org/
+
+DolphinScheduler Resources:
+- Issue: https://github.com/apache/dolphinscheduler/issues/
+- Mailing list: dev@dolphinscheduler.apache.org
+- Documents: https://dolphinscheduler.apache.org/zh-cn/docs/<VERSION>/about/introduction
+```

docs/docs/en/development/have-questions.md

@@ -1,65 +0,0 @@
-# Have Questions?
-
-## StackOverflow
-
-For usage questions, it is recommended you use the StackOverflow tag [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) as it is an active forum for DolphinScheduler users’ questions and answers.
-
-Some quick tips when using StackOverflow:
-
-- Prior to asking submitting questions, please:
-  - Search StackOverflow’s [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) tag to see if your question has already been answered
-- Please follow the StackOverflow [code of conduct](https://stackoverflow.com/help/how-to-ask)
-- Always use the apache-dolphinscheduler tag when asking questions
-- Please do not cross-post between [StackOverflow](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) and [GitHub issues](https://github.com/apache/dolphinscheduler/issues/new/choose)
-
-Question template:
-
-> **Describe the question**
->
-> A clear and concise description of what the question is.
->
-> **Which version of DolphinScheduler:**
->
->  -[1.3.0-preview]
->
-> **Additional context**
->
-> Add any other context about the problem here.
->
-> **Requirement or improvement**
->
-> \- Please describe about your requirements or improvement suggestions.
-
-For broad, opinion based, ask for external resources, debug issues, bugs, contributing to the project, and scenarios, it is recommended you use the[ GitHub issues ](https://github.com/apache/dolphinscheduler/issues/new/choose)or dev@dolphinscheduler.apache.org mailing list.
-
-## Mailing Lists
-
-- [dev@dolphinscheduler.apache.org](https://lists.apache.org/list.html?dev@dolphinscheduler.apache.org) is for people who want to contribute code to DolphinScheduler. [(subscribe)](mailto:dev-subscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20subscribe)) [(unsubscribe)](mailto:dev-unsubscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20unsubscribe)) [(archives)](http://lists.apache.org/list.html?dev@dolphinscheduler.apache.org)
-
-Some quick tips when using email:
-
-- Prior to asking submitting questions, please:
-  - Search StackOverflow at [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) to see if your question has already been answered
-
-- Tagging the subject line of your email will help you get a faster response, e.g. [api-server]: How to get open api interface?
-
-- Tags may help identify a topic by:
-  - Component: MasterServer,ApiServer,WorkerServer,AlertServer, etc
-  - Level: Beginner, Intermediate, Advanced
-  - Scenario: Debug, How-to
-
-- For error logs or long code examples, please use [GitHub gist](https://gist.github.com/) and include only a few lines of the pertinent code / log within the email.
-
-## Chat Rooms
-
-Chat rooms are great for quick questions or discussions on specialized topics. 
-
-The following chat rooms are officially part of Apache DolphinScheduler:
-
-​	The Slack workspace URL: http://asf-dolphinscheduler.slack.com/.
-
-​	You can join through invitation url: https://s.apache.org/dolphinscheduler-slack. 
-
-This chat room is used for questions and discussions related to using DolphinScheduler.
-
- 

docs/docs/en/faq.md

@@ -23,7 +23,8 @@ Note：**Due to the large number of services, it is recommended that the single-
 
 ## Q: Which mailboxes does the system support?
 
-A: Support most mailboxes, qq, 163, 126, 139, outlook, aliyun, etc. are supported. Support TLS and SSL protocols, optionally configured in alert.properties
+A: Support most mailboxes, qq, 163, 126, 139, outlook, aliyun, etc. are supported. Support TLS and SSL protocols, these can be configured in the ui of dolphinscheduler:
+[How to setup email alarm](../en/guide/alert/email.md).
 
 ---
 
@@ -125,7 +126,7 @@ A:   1, if the replacement variable contains special characters, **use the \ tra
 
 ​       4, monitorServerState = "false", whether the service monitoring script is started, the default is not to start the service monitoring script. **If the service monitoring script is started, the master and worker services are monitored every 5 minutes, and if the machine is down, it will automatically restart.**
 
-​       5, hdfsStartupSate="false", whether to enable HDFS resource upload function. The default is not enabled. **If it is not enabled, the resource center cannot be used.** If enabled, you need to configure the configuration of fs.defaultFS and yarn in conf/common/hadoop/hadoop.properties. If you use namenode HA, you need to copy core-site.xml and hdfs-site.xml to the conf root directory.
+​       5, hdfsStartupSate="false", whether to enable HDFS resource upload function. The default is not enabled. **If it is not enabled, the resource center cannot be used.** If enabled, you need to configure the configuration of resource.hdfs.fs.defaultFS and yarn in conf/common/hadoop/hadoop.properties. If you use namenode HA, you need to copy core-site.xml and hdfs-site.xml to the conf root directory.
 
 ​    Note: **The 1.0.x version does not automatically create the hdfs root directory, you need to create it yourself, and you need to deploy the user with hdfs operation permission.**
 
@@ -300,9 +301,10 @@ change into
 ---
 
 ## Q : how to add a worker server
-A: 1, Create deployment user and hosts mapping, please refer 1.3 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/laster/user_doc/installation/cluster.html)
 
-​		2, Configure hosts mapping and ssh access and modify directory permissions. please refer 1.4 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/laster/user_doc/installation/cluster.html)
+A: 1, Create deployment user and hosts mapping, please refer 1.3 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/3.0.5/user_doc/installation/cluster)
+
+​ 2, Configure hosts mapping and ssh access and modify directory permissions. please refer 1.4 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/3.0.5/user_doc/installation/cluster)
 
 ​		3, Copy the deployment directory from worker server that has already deployed
 
@@ -445,12 +447,16 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory
 ```
 sudo rm -rf node_modules
 ```
-   ​	2, install node-sass through npm.taobao.org
+
- ```
+​	2, install node-sass through npm.taobao.org
- sudo npm uninstall node-sass
+
- sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
+```
- ```
+sudo npm uninstall node-sass
-   3, if the 2nd step failure, please, [referer url](https://dolphinscheduler.apache.org/en-us/development/frontend-development.html)
+sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
+```
+
+3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md)
+
 ```
  sudo npm rebuild node-sass
 ```
@@ -574,6 +580,12 @@ Modify the configuration in `common.properties`:
 # dolphin.scheduler.network.priority.strategy=default
 ```
 
+In addition, if you would like to obtain the ip address from the specified network card, modify the configuration `dolphin.scheduler.network.interface.preferred` in `common.properties`. For example, you could modify the configuration in `common.properties` as follows to obtain the ip address from the network card `eth1`:
+
+```shell
+dolphin.scheduler.network.interface.preferred=eth1
+```
+
 After configuration is modified, restart the service to activation
 
 If the ip address is still wrong, please download [dolphinscheduler-netutils.jar](/asset/dolphinscheduler-netutils.jar) to the machine, execute the following commands and feedback the output to the community developers:
@@ -700,47 +712,14 @@ A：The repair can be completed by executing the following SQL in the database:
 update t_ds_version set version='2.0.1';
 ```
 
-## Can not find python-gateway-server in distribute package
+---
 
-After version 3.0.0-alpha, Python gateway server integrate into API server, and Python gateway service will start when you
+## Q：Can not find python-gateway-server in distribute package
+
+A：After version 3.0.0-alpha, Python gateway server integrate into API server, and Python gateway service will start when you
 start API server. If you want disabled when Python gateway service you could change API server configuration in path
 `api-server/conf/application.yaml` and change attribute `python-gateway.enabled : false`.
 
-## How to Build Custom Docker Image
-
-DolphinScheduler will release new Docker images after it released, you could find them in DockerHub. You could create
-custom Docker images base on those images if you want to change image like add some dependencies or upgrade package.
-
-```Dockerfile
-FROM apache/dolphinscheduler-standalone-server
-RUN apt update ; \
-    apt install -y <YOUR-CUSTOM-DEPENDENCE> ; \
-```
-
-If you want to modify DolphinScheduler source code, then build and distribute your own images, you can run below command
-to build Docker images and install them locally, which you could find them by command `docker imaegs`.
-
-```shell
-./mvnw -B clean install \
-  -Dmaven.test.skip \
-  -Dmaven.javadoc.skip \
-  -Dmaven.checkstyle.skip \
-  -Dmaven.deploy.skip \
-  -Ddocker.tag=latest \
-  -Pdocker,release
-```
-
-If you want to modify DolphinScheduler source code, but also want to add customize dependencies of Docker image, you can
-modify the definition of Dockerfile after modifying the source code. You can run the following command in root source code
-directory to find all Dockerfile files.
-
-```shell
-find . -iname 'Dockerfile'
-```
-
-Then run the command above start with `./mvnw -B clean install`. You can see all docker images you just created with
-command `docker images` after finish commnand `mvnw`.
-
 ---
 
-## We will collect more FAQ later
+We will collect more FAQ later

docs/docs/en/guide/alert/alert_plugin_user_guide.md

@@ -1,18 +1,22 @@
 # Alert Component User Guide
 
-## How to Create Alert Plugins and Alert Groups
+## Alert Plugins and Alert Groups Creation
 
 In version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the task status, executes the alarm instance sending logic if it matches, and filters if it does not match. When create alert instances then associate them with alert groups. Alert group can use multiple alert instances.
 The alarm module supports the following scenarios:
-<img src="/img/alert/alert_scenarios_en.png">
 
-The steps to use are as follows:
+<img src="../../../../img/alert/alert_scenarios_en.png">
 
-First, go to the Security Center page. Select Alarm Group Management, click Alarm Instance Management on the left and create an alarm instance. Select the corresponding alarm plug-in and fill in the relevant alarm parameters.
+Steps to be used are as follows:
 
-Then select Alarm Group Management, create an alarm group, and choose the corresponding alarm instance.
+- Go to `Security -> Alarm Group Management -> Alarm Instance Management -> Alarm Instance`. 
+- Select the corresponding alarm plug-in and fill in the relevant alarm parameters.
+- Select `Alarm Group Management`, create an alarm group, and choose the corresponding alarm instance.
 
-<img src="/img/alert/alert_step_1.png">
+![alert-instance01](../../../../img/new_ui/dev/alert/alert_instance01.png)
-<img src="/img/alert/alert_step_2.png">
+
-<img src="/img/alert/alert_step_3.png">
+![alert-instance02](../../../../img/new_ui/dev/alert/alert_instance02.png)
-<img src="/img/alert/alert_step_4.png">
+
+![alert-instance03](../../../../img/new_ui/dev/alert/alert_instance03.png)
+
+![alert-instance04](../../../../img/new_ui/dev/alert/alert_instance04.png)

docs/docs/en/guide/alert/dingtalk.md

@@ -1,27 +1,27 @@
 # DingTalk
 
-If you need to use `DingTalk` for alerting, create an alert instance in the alert instance management and select the DingTalk plugin. 
+If you need to use `DingTalk` for alerting, create an alert instance in the alert instance management and select the `DingTalk` plugin.
+
 The following shows the `DingTalk` configuration example:
 
-![dingtalk-plugin](/img/alert/dingtalk-plugin.png)
+![alert-dingtalk](../../../../img/new_ui/dev/alert/alert_dingtalk.png)
 
 ## Parameter Configuration
 
-* Webhook
+| **Parameter** | **Description** |
-  > The format is: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX
+| --- | --- |
-* Keyword
+| Warning Type | Alert on success or failure or both. |
-  > Custom keywords for security settings
+| WebHook | The format is: [https://oapi.dingtalk.com/robot/send?access\_token=XXXXXX](https://oapi.dingtalk.com/robot/send?access_token=XXXXXX) |
-* Secret
+| Keyword | Custom keywords for security settings. |
-  > Signature of security settings
+| Secret | Signature of security settings   |
-* MessageType
+| Msg Type | Message parse type (support txt, markdown, markdownV2, html). |
-  > Support both text and markdown types
+| At User Mobile | When a custom bot sends a message, you can specify the "@person list" by their mobile phone number. When the selected people in the "@people list" receive the message, there will be a `@` message reminder. `No disturb` mode always receives reminders, and "someone @ you" appears in the message. The "At User Mobile" represents mobile phone number of the "@person" |
+| At User Ids | The user ID by "@person" |
+| Proxy | The proxy address of the proxy server. |
+| Port | The proxy port of Proxy-Server. |
+| User | Authentication(Username) for the proxy server. |
+| Password | Authentication(Password) for the proxy server. |
 
-When a custom bot sends a message, you can specify the "@person list" by their mobile phone number. When the selected people in the "@people list" receive the message, there will be a `@` message reminder. `No disturb` mode always receives reminders, and "someone @ you" appears in the message.
+## Reference
-* @Mobiles
-  > The mobile phone number of the "@person"
-* @UserIds
-  > The user ID by "@person"
-* @All
-  > @Everyone
 
-[DingTalk Custom Robot Access Development Documentation](https://open.dingtalk.com/document/robots/custom-robot-access)
+- [DingTalk Custom Robot Access Development Documentation](https://open.dingtalk.com/document/robots/custom-robot-access) 

docs/docs/en/guide/alert/email.md

@@ -0,0 +1,10 @@
+# Email  
+If you need to use `Email` for alerting, create an alert instance in the alert instance management and select the Email plugin.
+
+The following shows the `Email` configuration example:
+
+![alert-email](../../../../img/alert/email-alter-setup1-en.png)
+
+![alert-email](../../../../img/alert/email-alter-setup2-en.png)
+
+![alert-email](../../../../img/alert/email-alter-setup3-en.png)

docs/docs/en/guide/alert/enterprise-webexteams.md

@@ -1,37 +1,32 @@
 # Webex Teams
 
-If you need to use `Webex Teams` to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin.
+If you need to use `Webex Teams` to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin. You can pick private alert or room group chat alert.
-You can pick private alert or room group chat alert.
 The following is the `WebexTeams` configuration example:
 
-![enterprise-webexteams-plugin](/img/alert/enterprise-webexteams-plugin.png)
+![enterprise-webexteams-plugin](../../../../img/alert/enterprise-webexteams-plugin.png)
 
 ## Parameter Configuration
 
-* botAccessToken
+| **Parameter** | **Description** |
-  > The robot's access token
+| --- | --- |
-* roomID
+| botAccessToken | The access token of robot. |
-  > The ID of the room that receives message (only support one room ID)
+| roomID | The ID of the room that receives message (only support one room ID). |
-* toPersonId
+| toPersonId | The person ID of the recipient when sending a private 1:1 message. |
-  > The person ID of the recipient when sending a private 1:1 message
+| toPersonEmail | The email address of the recipient when sending a private 1:1 message. |
-* toPersonEmail
+| atSomeoneInRoom | If the message destination is room, the emails of the person being @, use `,` (eng commas) to separate multiple emails. |
-  > The email address of the recipient when sending a private 1:1 message
+| destination |The destination of the message (one message only support one destination). |
-* atSomeoneInRoom
-  > If the message destination is room, the emails of the person being @, use `,` (eng commas) to separate multiple emails
-* destination
-  > The destination of the message (one message only support one destination)
 
-## Create a Bot
+## Create Bot
 
 Create a bot visit [Official Website My-Apps](https://developer.webex.com/my-apps) to `Create a New APP` and select `Create a Bot`, fill in the bot information and acquire `bot username` and `bot ID` for further usage.
 
-![enterprise-webexteams-bot-info](/img/alert/enterprise-webexteams-bot.png)
+![enterprise-webexteams-bot-info](../../../../img/alert/enterprise-webexteams-bot.png)
 
 ## Create a Room
 
 Create a root visit [Official Website for Developer APIs](https://developer.webex.com/docs/api/v1/rooms/create-a-room) to create a new room, fill in the room name and acquire `id`(room ID) and `creatorId` for further usage.
 
-![enterprise-webexteams-room-info](/img/alert/enterprise-webexteams-room.png)
+![enterprise-webexteams-room-info](../../../../img/alert/enterprise-webexteams-room.png)
 
 ### Invite Bot to the Room
 
@@ -43,22 +38,24 @@ In this way, you can send private message to a person by `User Email` or `UserId
 The `user Email` is user register Email.
 The `userId` we can acquire it from the `creatorId` of creating a new group chat room API.
 
-![enterprise-webexteams-private-message-form](/img/alert/enterprise-webexteams-private-form.png)
+![enterprise-webexteams-private-message-form](../../../../img/alert/enterprise-webexteams-private-form.png)
 
 ### Private Alert Message Example
 
-![enterprise-webexteams-private-message-example](/img/alert/enterprise-webexteams-private-msg.png)
+![enterprise-webexteams-private-message-example](../../../../img/alert/enterprise-webexteams-private-msg.png)
 
 ## Send Group Room Message
 
 In this way, you can send group room message to a room by `Room ID`. Fill in the `Room Id` and `Bot Access Token` and select `Destination` `roomId`.
 The `Room ID` we can acquire it from the `id` of creating a new group chat room API.
 
-![enterprise-webexteams-room](/img/alert/enterprise-webexteams-group-form.png)
+![enterprise-webexteams-room](../../../../img/alert/enterprise-webexteams-group-form.png)
 
 ### Group Room Alert Message Example
 
-![enterprise-webexteams-room-message-example](/img/alert/enterprise-webexteams-room-msg.png)
+![enterprise-webexteams-room-message-example](../../../../img/alert/enterprise-webexteams-room-msg.png)
 
-[WebexTeams Application Bot Guide](https://developer.webex.com/docs/bots)
+## References:
-[WebexTeams Message Guide](https://developer.webex.com/docs/api/v1/messages/create-a-message)
+
+- [WebexTeams Application Bot Guide](https://developer.webex.com/docs/bots)
+- [WebexTeams Message Guide](https://developer.webex.com/docs/api/v1/messages/create-a-message)

docs/docs/en/guide/alert/enterprise-wechat.md

@@ -3,7 +3,7 @@
 If you need to use `Enterprise WeChat` to alert, create an alert instance in the alert instance management, and choose the `WeChat` plugin.
 The following is the `WeChat` configuration example:
 
-![enterprise-wechat-plugin](/img/alert/enterprise-wechat-plugin.png)
+![enterprise-wechat-plugin](../../../../img/alert/enterprise-wechat-plugin.png)
 
 ## Send Type
 
@@ -14,15 +14,15 @@ The parameter `send.type` corresponds to sending messages to Enterprise WeChat c
 The APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending messages to both specified users and all members. Currently, send to specified enterprise department and tags are not supported, a new PR to contribute is welcomed.
 The following is the `APP` alert config example:
 
-![enterprise-wechat-app-msg-config](/img/alert/wechat-app-form-example.png)
+![enterprise-wechat-app-msg-config](../../../../img/alert/wechat-app-form-example.png)
 
 The following is the `APP` `MARKDOWN` alert message example:
 
-![enterprise-wechat-app-msg-markdown](/img/alert/enterprise-wechat-app-msg-md.png)
+![enterprise-wechat-app-msg-markdown](../../../../img/alert/enterprise-wechat-app-msg-md.png)
 
 The following is the `APP` `TEXT` alert message example:
 
-![enterprise-wechat-app-msg-text](/img/alert/enterprise-wechat-app-msg.png)
+![enterprise-wechat-app-msg-text](../../../../img/alert/enterprise-wechat-app-msg.png)
 
 #### Prerequisites
 
@@ -34,7 +34,7 @@ The Enterprise WeChat APPs support sending messages to both specified users and
 To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
 The following is the `query userId` API example:
 
-![enterprise-wechat-create-group](/img/alert/enterprise-wechat-query-userid.png)
+![enterprise-wechat-create-group](../../../../img/alert/enterprise-wechat-query-userid.png)
 
 #### References
 
@@ -46,15 +46,15 @@ APP: https://work.weixin.qq.com/api/doc/90000/90135/90236
 The Group Chat send type means to notify the alert results via group chat created by Enterprise WeChat API, sending messages to all members of the group and specified users are not supported.
 The following is the `Group Chat` alert config example:
 
-![enterprise-wechat-app-msg-config](/img/alert/wechat-group-form-example.png)
+![enterprise-wechat-app-msg-config](../../../../img/alert/wechat-group-form-example.png)
 
 The following is the `APP` `MARKDOWN` alert message example:
 
-![enterprise-wechat-group-msg-markdown](/img/alert/enterprise-wechat-group-msg-md.png)
+![enterprise-wechat-group-msg-markdown](../../../../img/alert/enterprise-wechat-group-msg-md.png)
 
 The following is the `Group Chat` `TEXT` alert message example:
 
-![enterprise-wechat-group-msg-text](/img/alert/enterprise-wechat-group-msg.png)
+![enterprise-wechat-group-msg-text](../../../../img/alert/enterprise-wechat-group-msg.png)
 
 #### Prerequisites
 
@@ -62,10 +62,10 @@ Before sending messages to group chat, create a new group chat by Enterprise WeC
 To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
 The following is the `create new group chat` API and `query userId` API example:
 
-![enterprise-wechat-create-group](/img/alert/enterprise-wechat-create-group.png)
+![enterprise-wechat-create-group](../../../../img/alert/enterprise-wechat-create-group.png)
 
-![enterprise-wechat-create-group](/img/alert/enterprise-wechat-query-userid.png)
+![enterprise-wechat-create-group](../../../../img/alert/enterprise-wechat-query-userid.png)
 
-#### References
+## Reference
 
-Group Chat：https://work.weixin.qq.com/api/doc/90000/90135/90248
+- Group Chat：https://work.weixin.qq.com/api/doc/90000/90135/90248

docs/docs/en/guide/alert/feishu.md

@@ -0,0 +1,17 @@
+# Feishu
+
+If you need to use `Feishu` for alerting, create an alert instance in the alert instance management and select
+the `Feishu` plugin.
+
+The following shows the `Feishu` configuration example:
+
+![alert-feishu](../../../../img/new_ui/dev/alert/alert_feishu.png)
+
+## Parameter Configuration
+
+* Webhook
+  > Copy the robot webhook URL shown below:
+
+  ![alert-feishu-webhook](../../../../img/new_ui/dev/alert/alert_feishu_webhook.png)
+
+[Feishu：Use bots in groups](https://www.feishu.cn/hc/en-US/articles/360024984973)

docs/docs/en/guide/alert/http.md

@@ -4,31 +4,28 @@ If you need to use `Http script` for alerting, create an alert instance in the a
 
 ## Parameter Configuration
 
-* URL
+| **Parameter** | **Description** |
-  > The `Http` request URL needs to contain protocol, host, path and parameters if the method is `GET`
+| --- | --- |
-* Request Type
+| URL | The `Http` request URL needs to contain protocol, host, path and parameters if the method is `GET`. |
-  > Select the request type from `POST` or `GET`
+| Request Type | Select the request type from `POST` or `GET`. |
-* Headers
+| Headers | The headers of the `Http` request in JSON format. |
-  > The headers of the `Http` request in JSON format
+| Body | The request body of the `Http` request in JSON format, when using `POST` method to alert. |
-* Body
+| Content Field | The field name to place the alert information. |
-  > The request body of the `Http` request in JSON format, when using `POST` method to alert
-* Content Field
-  > The field name to place the alert information
 
 ## Send Type
 
 Using `POST` and `GET` method to send `Http` request in the `Request Type`.
 
-### GET Http
+### GET HTTP
 
 Send alert information by `Http` GET method.
 The following shows the `GET` configuration example:
 
-![enterprise-wechat-app-msg-config](/img/alert/http-get-example.png)
+![enterprise-wechat-app-msg-config](../../../../img/alert/http-get-example.png)
 
-### POST Http
+### POST HTTP
 
 Send alert information inside `Http` body by `Http` POST method.
 The following shows the `POST` configuration example:
 
-![enterprise-wechat-app-msg-config](/img/alert/http-post-example.png)
+![enterprise-wechat-app-msg-config](../../../../img/alert/http-post-example.png)

docs/docs/en/guide/alert/script.md

@@ -1,17 +1,18 @@
-# DingTalk
+# Script
 
 If you need to use `Shell script` for alerting, create an alert instance in the alert instance management and select the `Script` plugin. 
 The following shows the `Script` configuration example:
 
-![dingtalk-plugin](/img/alert/script-plugin.png)
+![dingtalk-plugin](../../../../img/alert/script-plugin.png)
 
 ## Parameter Configuration
 
-* User Params
+| **Parameter** | **Description** |
-  > User defined parameters will pass to the script.
+| --- | --- |
-* Script Path
+| User Params | User defined parameters will pass to the script. |
-  > The file location path in the server.
+| Script Path |The file location path in the server. |
-* Type
+| Type | Support `Shell` script. |
-  > Support `Shell` script.
 
-**_Notice:_** please consider the script file access privileges with the executing tenant.
+### Note
+
+Consider the script file access privileges with the executing tenant.

docs/docs/en/guide/alert/slack.md

@@ -0,0 +1,47 @@
+# Slack
+
+If you need to use `Slack` for alerting, create an alert instance in the alert instance management and select
+the `Slack` plugin.
+
+The following shows the `Slack` configuration example:
+
+![alert-slack](../../../../img/new_ui/dev/alert/alert_slack.png)
+
+## Parameter Configuration
+
+* Webhook
+
+  > Copy the `Incoming Webhooks` address from the APP, check the image below:
+
+  ![alert-slack-webhook](../../../../img/new_ui/dev/alert/alert_slack_webhook_addr.png)
+
+* UserName
+
+  > (Deprecated) The sender name. Currently, Slack use APP as the sender name due to the Slack updates.
+
+## How to Get Webhook
+
+Please refer to the official article [Slack: Sending messages using Incoming Webhooks](https://api.slack.com/messaging/webhooks) to create new webhooks.
+
+### Create a new Slack APP
+
+Visit [Slack Official Website](https://api.slack.com/apps/new) and create a new APP.
+
+### Activate Incoming Webhooks Setting
+
+When finish creating a new APP, select `Incoming Webhooks` of the column `Feature` on the left of the APP page, and switch `Activate Incoming Webhooks` to `ON`.
+![alert-slack-webhook](../../../../img/new_ui/dev/alert/alert_slack_enable.png)
+
+### Create a New Incoming Webhook
+
+Click `Add New Webhook to Workspace` and select a group to post messages.
+![select-post-group](../../../../img/new_ui/dev/alert/alert_slack_new.png)
+
+### Acquire Incoming Webhooks Address
+
+Copy the `Incoming Webhooks` address to DolphinScheduler, check the image below:
+The default format of the `Incoming Webhooks`: `https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX`
+
+![alert-slack-webhook](../../../../img/new_ui/dev/alert/alert_slack_webhook_addr.png)
+
+Reference：[Slack：Sending messages using Incoming Webhooks](https://api.slack.com/messaging/webhooks)

docs/docs/en/guide/alert/telegram.md

@@ -3,30 +3,25 @@
 If you need `Telegram` to alert, create an alert instance in the alert instance management, and choose the `Telegram` plugin.
 The following shows the `Telegram` configuration example:
 
-![telegram-plugin](/img/alert/telegram-plugin.png)
+![alert-telegram](../../../../img/new_ui/dev/alert/alert_telegram.png)
 
 ## Parameter Configuration
 
-* WebHook:
+| **Parameter** | **Description** |
-  > The WebHook of Telegram when use robot to send message
+| --- | --- |
-* botToken
+| WebHook | The WebHook of Telegram when use robot to send message. |
-  > The robot's access token
+| botToken | The access token of robot. |
-* chatId
+| chatId | Sub Telegram Channel. |
-  > Sub Telegram Channel
+| parseMode | Message parse type (support txt, markdown, markdownV2, html). |
-* parseMode
+| EnableProxy | Enable proxy sever. |
-  > Message parse type (support txt, markdown, markdownV2, html)
+| Proxy | The proxy address of the proxy server. |
-* EnableProxy
+| Port | The proxy port of proxy server. |
-  > Enable proxy sever
+| User | Authentication(Username) for the proxy server. |
-* Proxy
+| Password | Authentication(Password) for the proxy server. |
-  > the proxy address of the proxy server
-* Port
-  > the proxy port of Proxy-Server
-* User
-  > Authentication(Username) for the proxy server
-* Password
-  > Authentication(Password) for the proxy server
 
-**NOTICE**：The webhook needs to be able to receive and use the same JSON body of HTTP POST that DolphinScheduler constructs and the following shows the JSON body:
+### NOTE
+
+The webhook needs to be able to receive and use the same JSON body of HTTP POST that DolphinScheduler constructs and the following shows the JSON body:
 
 ```json
 {
@@ -35,8 +30,8 @@ The following shows the `Telegram` configuration example:
 }
 ```
 
-References:
+## References:
+
 - [Telegram Application Bot Guide](https://core.telegram.org/bots)
 - [Telegram Bots Api](https://core.telegram.org/bots/api)
-- [Telegram SendMessage Api](https://core.telegram.org/bots/api#sendmessage)
+- [Telegram SendMessage Api](https://core.telegram.org/bots/api#sendmessage)
-

docs/docs/en/guide/api/healthcheck.md

@@ -0,0 +1,41 @@
+# Health Check
+
+## Background
+
+Health check are designed to provide a unique way to check the health of the dolphinscheduler service. It includes the health status of modules, such as DB, cache, network, etc.
+
+## Endpoint
+
+### API-Server
+
+```shell
+curl --request GET 'http://localhost:12345/dolphinscheduler/actuator/health'
+
+{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2","validationQuery":"isValid()"}}}}
+```
+
+### Master-Server
+
+```shell
+curl --request GET 'http://localhost:5679/actuator/health'
+
+{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2","validationQuery":"isValid()"}}}}
+```
+
+### Worker-Server
+
+```shell
+curl --request GET 'http://localhost:1235/actuator/health'
+
+{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2","validationQuery":"isValid()"}}}}
+```
+
+### Alert-Server
+
+```shell
+curl --request GET 'http://localhost:50053/actuator/health'
+
+{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2","validationQuery":"isValid()"}}}}
+```
+
+> Notice: If you modify the default service port and address, you need to modify the IP+Port to the modified value.

docs/docs/en/guide/api/open-api.md

@@ -0,0 +1,68 @@
+# Open API
+
+## Background
+
+Generally, projects and processes are created through pages, but considering the integration with third-party systems requires API calls to manage projects and workflows.
+
+## The Operation Steps of DolphinScheduler API Calls
+
+### Create a Token
+
+1. Log in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.
+
+![create-token](../../../../img/new_ui/dev/security/create-token.png)
+
+2. Select the "Expiration time" (Token validity time), select "User" (choose the specified user to perform the API operation), click "Generate token", copy the `Token` string, and click "Submit".
+
+![token-expiration](../../../../img/new_ui/dev/open-api/token_expiration.png)
+
+### Examples
+
+#### Query project list
+
+1. Open the API documentation 
+
+> Address：http://{API server ip}:12345/dolphinscheduler/doc.html?language=en_US&lang=en
+
+![api-doc](../../../../img/new_ui/dev/open-api/api_doc.png)
+
+2. select a test API, the API selected for this test is `queryAllProjectList`
+
+    > projects/list
+                                                                                  >
+3. Open `Postman`, fill in the API address, enter the `Token` in `Headers`, and then send the request to view the result:
+    
+   ```
+    token: The Token just generated
+    ```
+   
+![api-test](../../../../img/new_ui/dev/open-api/api_test.png)
+
+#### Create a Project
+
+This demonstrates how to use the calling api to create the corresponding project.
+
+By consulting the api documentation, configure the KEY as Accept and VALUE as the parameter of application/json in the headers of Postman.
+
+![create-project01](../../../../img/new_ui/dev/open-api/create_project01.png)
+
+And then configure the required projectName and description parameters in Body.
+
+![create-project02](../../../../img/new_ui/dev/open-api/create_project02.png)
+
+Check the post request result.
+
+![create-project03](../../../../img/new_ui/dev/open-api/create_project03.png)
+
+The returned `msg` information is "success", indicating that we have successfully created the project through API.
+
+If you are interested in the source code of creating a project, please continue to read the following：
+
+### Appendix： The Source Code of Creating a Project
+
+![api-source01](../../../../img/new_ui/dev/open-api/api_source01.png)
+
+![api-source02](../../../../img/new_ui/dev/open-api/api_source02.png)
+
+
+