[Imporvement] Improve the MasterCofnig module comments of dolphinscheduler (#16134)

Co-authored-by: xiangzihao <460888207@qq.com>

.github/workflows/backend.yml

@@ -153,7 +153,7 @@ jobs:
       fail-fast: false
       matrix:
         db: ["mysql", "postgresql"]
-        version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"]
+        version: ["3.1.9", "3.2.0"]
     steps:
       - name: Set up JDK 8
         uses: actions/setup-java@v2

deploy/kubernetes/dolphinscheduler/README.md

@@ -120,6 +120,12 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
 | conf.common."alert.rpc.port" | int | `50052` | rpc port |
 | conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
+| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` |  |
+| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
+| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
+| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
+| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
+| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required |
 | conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
 | conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
 | conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
@@ -138,11 +144,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
 | conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
 | conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
-| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
-| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
-| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
-| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
-| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
 | conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
 | conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
 | conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
@@ -158,6 +159,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | conf.common."yarn.application.status.address" | string | `"http://ds1:%s/ws/v1/cluster/apps/%s"` | if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname |
 | conf.common."yarn.job.history.status.address" | string | `"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"` | job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) |
 | conf.common."yarn.resourcemanager.ha.rm.ids" | string | `"192.168.xx.xx,192.168.xx.xx"` | if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty |
+| datasource.profile | string | `"postgresql"` | The profile of datasource |
 | externalDatabase.database | string | `"dolphinscheduler"` | The database of external database |
 | externalDatabase.driverClassName | string | `"org.postgresql.Driver"` | The driverClassName of external database |
 | externalDatabase.enabled | bool | `false` | If exists external database, and set postgresql.enable value to false. external database will be used, otherwise Dolphinscheduler's internal database will be used. |
@@ -306,7 +308,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
 | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. |
 | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. |
 | worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
-| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. |
 | worker.keda.advanced | object | `{}` | Specify HPA related options |
 | worker.keda.cooldownPeriod | int | `30` | How many seconds KEDA will wait before scaling to zero. Note that HPA has a separate cooldown period for scale-downs |
 | worker.keda.enabled | bool | `false` | Enable or disable the Keda component |

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -51,7 +51,6 @@ Create a default common labels.
 {{- define "dolphinscheduler.common.labels" -}}
 app.kubernetes.io/instance: {{ .Release.Name }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
-app.kubernetes.io/version: {{ .Chart.AppVersion }}
 {{- end -}}
 
 {{/*

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if and .Values.alert.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-alert
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
+    {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
+data:
+  application.yaml: |
+    spring:
+      profiles:
+        active: {{ .Values.datasource.profile }}
+      jackson:
+        time-zone: UTC
+        date-format: "yyyy-MM-dd HH:mm:ss"
+      banner:
+        charset: UTF-8
+      datasource:
+        driver-class-name: org.postgresql.Driver
+        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+        username: root
+        password: root
+        hikari:
+          connection-test-query: select 1
+          pool-name: DolphinScheduler
+    
+    # Mybatis-plus configuration, you don't need to change it
+    mybatis-plus:
+      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
+      type-aliases-package: org.apache.dolphinscheduler.dao.entity
+      configuration:
+        cache-enabled: false
+        call-setters-on-nulls: true
+        map-underscore-to-camel-case: true
+        jdbc-type-for-null: NULL
+      global-config:
+        db-config:
+          id-type: auto
+        banner: false
+    
+    server:
+      port: 50053
+    
+    management:
+      endpoints:
+        web:
+          exposure:
+            include: health,metrics,prometheus
+      endpoint:
+        health:
+          enabled: true
+          show-details: always
+      health:
+        db:
+          enabled: true
+        defaults:
+          enabled: false
+      metrics:
+        tags:
+          application: ${spring.application.name}
+    
+    alert:
+      port: 50052
+      # Mark each alert of alert server if late after x milliseconds as failed.
+      # Define value is (0 = infinite), and alert server would be waiting alert result.
+      wait-timeout: 0
+      max-heartbeat-interval: 60s
+      query_alert_threshold: 100
+    
+    registry:
+      type: zookeeper
+      zookeeper:
+        namespace: dolphinscheduler
+        connect-string: localhost:2181
+        retry-policy:
+          base-sleep-time: 60ms
+          max-sleep: 300ms
+          max-retries: 5
+        session-timeout: 30s
+        connection-timeout: 9s
+        block-until-connected: 600ms
+        digest: ~
+    
+    metrics:
+      enabled: true
+    
+    # Override by profile
+    ---
+    spring:
+      config:
+        activate:
+          on-profile: mysql
+      datasource:
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+        username: root
+        password: root
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml

@@ -0,0 +1,258 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if and .Values.api.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-api
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
+    {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
+data:
+  application.yaml: |
+    server:
+      port: 12345
+      servlet:
+        session:
+          timeout: 120m
+        context-path: /dolphinscheduler/
+      compression:
+        enabled: true
+        mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
+      jetty:
+        max-http-form-post-size: 5000000
+        accesslog:
+          enabled: true
+          custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
+    
+    spring:
+      profiles:
+        active: {{ .Values.datasource.profile }}
+      banner:
+        charset: UTF-8
+      jackson:
+        time-zone: UTC
+        date-format: "yyyy-MM-dd HH:mm:ss"
+      servlet:
+        multipart:
+          max-file-size: 1024MB
+          max-request-size: 1024MB
+      messages:
+        basename: i18n/messages
+      datasource:
+        driver-class-name: org.postgresql.Driver
+        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+        username: root
+        password: root
+        hikari:
+          connection-test-query: select 1
+          pool-name: DolphinScheduler
+      quartz:
+        auto-startup: false
+        job-store-type: jdbc
+        jdbc:
+          initialize-schema: never
+        properties:
+          org.quartz.jobStore.isClustered: true
+          org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
+          org.quartz.scheduler.instanceId: AUTO
+          org.quartz.jobStore.tablePrefix: QRTZ_
+          org.quartz.jobStore.acquireTriggersWithinLock: true
+          org.quartz.scheduler.instanceName: DolphinScheduler
+          org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
+          org.quartz.jobStore.useProperties: false
+          org.quartz.jobStore.misfireThreshold: 60000
+          org.quartz.scheduler.makeSchedulerThreadDaemon: true
+          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+          org.quartz.jobStore.clusterCheckinInterval: 5000
+          org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
+      mvc:
+        pathmatch:
+          matching-strategy: ANT_PATH_MATCHER
+        static-path-pattern: /static/**
+    springdoc:
+      swagger-ui:
+        path: /swagger-ui.html
+      packages-to-scan: org.apache.dolphinscheduler.api
+    
+    # Mybatis-plus configuration, you don't need to change it
+    mybatis-plus:
+      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
+      type-aliases-package: org.apache.dolphinscheduler.dao.entity
+      configuration:
+        cache-enabled: false
+        call-setters-on-nulls: true
+        map-underscore-to-camel-case: true
+        jdbc-type-for-null: NULL
+      global-config:
+        db-config:
+          id-type: auto
+        banner: false
+    
+    management:
+      endpoints:
+        web:
+          exposure:
+            include: health,metrics,prometheus
+      endpoint:
+        health:
+          enabled: true
+          show-details: always
+      health:
+        db:
+          enabled: true
+        defaults:
+          enabled: false
+      metrics:
+        tags:
+          application: ${spring.application.name}
+    
+    registry:
+      type: zookeeper
+      zookeeper:
+        namespace: dolphinscheduler
+        connect-string: localhost:2181
+        retry-policy:
+          base-sleep-time: 60ms
+          max-sleep: 300ms
+          max-retries: 5
+        session-timeout: 60s
+        connection-timeout: 15s
+        block-until-connected: 15s
+        digest: ~
+    
+    api:
+      audit-enable: false
+      # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
+      # global max request number per second
+      # default tenant-level max request number
+      traffic-control:
+        global-switch: false
+        max-global-qps-rate: 300
+        tenant-switch: false
+        default-tenant-qps-rate: 10
+          #customize-tenant-qps-rate:
+        # eg.
+        #tenant1: 11
+        #tenant2: 20
+      python-gateway:
+        # Weather enable python gateway server or not. The default value is false.
+        enabled: false
+        # Authentication token for connection from python api to python gateway server. Should be changed the default value
+        # when you deploy in public network.
+        auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
+        # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+        # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+        gateway-server-address: 0.0.0.0
+        # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+        # Python API side.
+        gateway-server-port: 25333
+        # The address of Python callback client.
+        python-address: 127.0.0.1
+        # The port of Python callback client.
+        python-port: 25334
+        # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+        # and socket server would never close even though no requests accept
+        connect-timeout: 0
+        # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+        # (0 = infinite), and socket server would never close even though no requests accept
+        read-timeout: 0
+    
+    metrics:
+      enabled: true
+    
+    security:
+      authentication:
+        # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
+        type: PASSWORD
+        # IF you set type `LDAP`, below config will be effective
+        ldap:
+          # ldap server config
+          urls: ldap://ldap.forumsys.com:389/
+          base-dn: dc=example,dc=com
+          username: cn=read-only-admin,dc=example,dc=com
+          password: password
+          user:
+            # admin userId when you use LDAP login
+            admin: read-only-admin
+            identity-attribute: uid
+            email-attribute: mail
+            # action when ldap user is not exist (supported types: CREATE,DENY)
+            not-exist-action: CREATE
+          ssl:
+            enable: false
+            # jks file absolute path && password
+            trust-store: "/ldapkeystore.jks"
+            trust-store-password: "password"
+        casdoor:
+          user:
+            admin: ""
+        oauth2:
+          enable: false
+          provider:
+            github:
+              authorizationUri: ""
+              redirectUri: ""
+              clientId: ""
+              clientSecret: ""
+              tokenUri: ""
+              userInfoUri: ""
+              callbackUrl: ""
+              iconUri: ""
+              provider: github
+            google:
+              authorizationUri: ""
+              redirectUri: ""
+              clientId: ""
+              clientSecret: ""
+              tokenUri: ""
+              userInfoUri: ""
+              callbackUrl: ""
+              iconUri: ""
+              provider: google
+    casdoor:
+      # Your Casdoor server url
+      endpoint: ""
+      client-id: ""
+      client-secret: ""
+      # The certificate may be multi-line, you can use `|-` for ease
+      certificate: ""
+      # Your organization name added in Casdoor
+      organization-name: ""
+      # Your application name added in Casdoor
+      application-name: ""
+      # Doplhinscheduler login url
+      redirect-url: ""
+    
+    # Override by profile
+    ---
+    spring:
+      config:
+        activate:
+          on-profile: mysql
+      datasource:
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+        username: root
+        password: root
+      quartz:
+        properties:
+          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
+{{- end }}
+
+
+

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml

@@ -0,0 +1,173 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if and .Values.master.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-master
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
+    {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
+data:
+  application.yaml: |
+    spring:
+      profiles:
+        active: {{ .Values.datasource.profile }}
+      banner:
+        charset: UTF-8
+      jackson:
+        time-zone: UTC
+        date-format: "yyyy-MM-dd HH:mm:ss"
+      datasource:
+        driver-class-name: org.postgresql.Driver
+        url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+        username: root
+        password: root
+        hikari:
+          connection-test-query: select 1
+          pool-name: DolphinScheduler
+      quartz:
+        job-store-type: jdbc
+        jdbc:
+          initialize-schema: never
+        properties:
+          org.quartz.threadPool.threadPriority: 5
+          org.quartz.jobStore.isClustered: true
+          org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
+          org.quartz.scheduler.instanceId: AUTO
+          org.quartz.jobStore.tablePrefix: QRTZ_
+          org.quartz.jobStore.acquireTriggersWithinLock: true
+          org.quartz.scheduler.instanceName: DolphinScheduler
+          org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
+          org.quartz.jobStore.useProperties: false
+          org.quartz.threadPool.makeThreadsDaemons: true
+          org.quartz.threadPool.threadCount: 25
+          org.quartz.jobStore.misfireThreshold: 60000
+          org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
+          org.quartz.scheduler.makeSchedulerThreadDaemon: true
+          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+          org.quartz.jobStore.clusterCheckinInterval: 5000
+    
+    # Mybatis-plus configuration, you don't need to change it
+    mybatis-plus:
+      mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
+      type-aliases-package: org.apache.dolphinscheduler.dao.entity
+      configuration:
+        cache-enabled: false
+        call-setters-on-nulls: true
+        map-underscore-to-camel-case: true
+        jdbc-type-for-null: NULL
+      global-config:
+        db-config:
+          id-type: auto
+        banner: false
+    
+    
+    registry:
+      type: zookeeper
+      zookeeper:
+        namespace: dolphinscheduler
+        connect-string: localhost:2181
+        retry-policy:
+          base-sleep-time: 60ms
+          max-sleep: 300ms
+          max-retries: 5
+        session-timeout: 30s
+        connection-timeout: 9s
+        block-until-connected: 600ms
+        digest: ~
+    
+    master:
+      listen-port: 5678
+      # master fetch command num
+      fetch-command-num: 10
+      # master prepare execute thread number to limit handle commands in parallel
+      pre-exec-threads: 10
+      # master execute thread number to limit process instances in parallel
+      exec-threads: 100
+      # master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s.
+      dispatch-task-number: 3
+      # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
+      host-selector: lower_weight
+      # master heartbeat interval
+      max-heartbeat-interval: 10s
+      # master commit task retry times
+      task-commit-retry-times: 5
+      # master commit task interval
+      task-commit-interval: 1s
+      state-wheel-interval: 5s
+      server-load-protection:
+        # If set true, will open master overload protection
+        enabled: true
+        # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
+        max-system-cpu-usage-percentage-thresholds: 0.7
+        # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
+        max-jvm-cpu-usage-percentage-thresholds: 0.7
+        # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
+        max-system-memory-usage-percentage-thresholds: 0.7
+        # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
+        max-disk-usage-percentage-thresholds: 0.7
+      # failover interval, the unit is minute
+      failover-interval: 10m
+      # kill yarn / k8s application when failover taskInstance, default true
+      kill-application-when-task-failover: true
+      registry-disconnect-strategy:
+        # The disconnect strategy: stop, waiting
+        strategy: waiting
+        # The max waiting time to reconnect to registry if you set the strategy to waiting
+        max-waiting-time: 100s
+      worker-group-refresh-interval: 10s
+    
+    server:
+      port: 5679
+    
+    management:
+      endpoints:
+        web:
+          exposure:
+            include: health,metrics,prometheus
+      endpoint:
+        health:
+          enabled: true
+          show-details: always
+      health:
+        db:
+          enabled: true
+        defaults:
+          enabled: false
+      metrics:
+        tags:
+          application: ${spring.application.name}
+    
+    metrics:
+      enabled: true
+    
+    # Override by profile
+    ---
+    spring:
+      config:
+        activate:
+          on-profile: mysql
+      datasource:
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+        username: root
+        password: root
+      quartz:
+        properties:
+          org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if and .Values.worker.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-worker
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
+    {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
+data:
+  application.yaml: |
+    spring:
+      banner:
+        charset: UTF-8
+      jackson:
+        time-zone: UTC
+        date-format: "yyyy-MM-dd HH:mm:ss"
+      autoconfigure:
+        exclude:
+          - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
+    
+    registry:
+      type: zookeeper
+      zookeeper:
+        namespace: dolphinscheduler
+        connect-string: localhost:2181
+        retry-policy:
+          base-sleep-time: 60ms
+          max-sleep: 300ms
+          max-retries: 5
+        session-timeout: 30s
+        connection-timeout: 9s
+        block-until-connected: 600ms
+        digest: ~
+    
+    worker:
+      # worker listener port
+      listen-port: 1234
+      # worker execute thread number to limit task instances in parallel
+      exec-threads: 100
+      # worker heartbeat interval
+      max-heartbeat-interval: 10s
+      # worker host weight to dispatch tasks, default value 100
+      host-weight: 100
+      server-load-protection:
+        # If set true, will open worker overload protection
+        enabled: true
+        # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
+        max-system-cpu-usage-percentage-thresholds: 0.7
+        # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
+        max-jvm-cpu-usage-percentage-thresholds: 0.7
+        # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
+        max-system-memory-usage-percentage-thresholds: 0.7
+        # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
+        max-disk-usage-percentage-thresholds: 0.7
+      registry-disconnect-strategy:
+        # The disconnect strategy: stop, waiting
+        strategy: waiting
+        # The max waiting time to reconnect to registry if you set the strategy to waiting
+        max-waiting-time: 100s
+      task-execute-threads-full-policy: REJECT
+      tenant-config:
+        # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
+        auto-create-tenant-enabled: true
+        # Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants.
+        distributed-tenant-enabled: false
+        # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
+        default-tenant-enabled: false
+    
+    server:
+      port: 1235
+    
+    management:
+      endpoints:
+        web:
+          exposure:
+            include: health,metrics,prometheus
+      endpoint:
+        health:
+          enabled: true
+          show-details: always
+      health:
+        db:
+          enabled: true
+        defaults:
+          enabled: false
+      metrics:
+        tags:
+          application: ${spring.application.name}
+    
+    metrics:
+      enabled: true
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@@ -37,7 +37,7 @@ data:
   {{- range $key, $value := index .Values.conf "common" }}
     {{- if and $.Values.minio.enabled }}
     {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
-    {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
+    {{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
     {{- end }}
     {{ $key }}={{ $value }}
   {{- end }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -115,6 +115,9 @@ spec:
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
               subPath: common.properties
+            - name: alert-config-volume
+              mountPath: /opt/dolphinscheduler/conf/application.yaml
+              subPath: application.yaml
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-alert
           {{- if .Values.alert.persistentVolumeClaim.enabled }}
@@ -126,4 +129,7 @@ spec:
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
+        - name: alert-config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-alert
 {{- end }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -121,6 +121,9 @@ spec:
               mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
               subPath: task-type-config.yaml
             {{- end }}
+            - name: api-config-volume
+              mountPath: /opt/dolphinscheduler/conf/application.yaml
+              subPath: application.yaml
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
@@ -136,6 +139,9 @@ spec:
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
+        - name: api-config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-api
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -109,6 +109,9 @@ spec:
           volumeMounts:
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-master
+            - name: master-config-volume
+              mountPath: /opt/dolphinscheduler/conf/application.yaml
+              subPath: application.yaml
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
@@ -122,6 +125,9 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
+        - name: master-config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-master
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         - name: config-volume
           configMap:

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -111,6 +111,9 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-worker-data
             - mountPath: "/opt/dolphinscheduler/logs"
               name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
+            - name: worker-config-volume
+              mountPath: /opt/dolphinscheduler/conf/application.yaml
+              subPath: application.yaml
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
               subPath: common.properties
@@ -139,6 +142,9 @@ spec:
         - name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
           emptyDir: {}
         {{- end }}
+        - name: worker-config-volume
+          configMap:
+            name: {{ include "dolphinscheduler.fullname" . }}-worker
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -49,6 +49,10 @@ image:
   # -- tools image
   tools: dolphinscheduler-tools
 
+datasource:
+  # -- The profile of datasource
+  profile: postgresql
+
 postgresql:
   # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL
   enabled: true
@@ -246,20 +250,25 @@ conf:
     # -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
     resource.storage.upload.base.path: /dolphinscheduler
 
-    # -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    resource.aws.access.key.id: minioadmin
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    aws.credentials.provider.type: AWSStaticCredentialsProvider
 
-    # -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    resource.aws.secret.access.key: minioadmin
+    # -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
+    aws.s3.access.key.id: minioadmin
 
-    # -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    resource.aws.region: ca-central-1
+    # -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
+    aws.s3.access.key.secret: minioadmin
+
+    # -- The AWS Region to use. if resource.storage.type=S3, This configuration is required
+    aws.s3.region: ca-central-1
 
     # -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-    resource.aws.s3.bucket.name: dolphinscheduler
+    aws.s3.bucket.name: dolphinscheduler
 
     # -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-    resource.aws.s3.endpoint: http://minio:9000
+    aws.s3.endpoint: http://minio:9000
 
     # -- alibaba cloud access key id, required if you set resource.storage.type=OSS
     resource.alibaba.cloud.access.key.id: <your-access-key-id>
@@ -645,8 +654,6 @@ worker:
     WORKER_HOST_WEIGHT: "100"
     # -- tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
     WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: true
-    # -- Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
-    WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT: false
     # -- If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`;
     DEFAULT_TENANT_ENABLED: false
 

docs/docs/en/architecture/configuration.md

@@ -191,19 +191,21 @@ The default configuration is as follows:
 
 Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
 
-For ETCD Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
-For JDBC Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
+For ETCD Registry, please see more details
+on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
+For JDBC Registry, please see more details
+on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
 
 ### common.properties [hadoop、s3、yarn config properties]
 
 Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
 
-|    Service    |           Configuration file           |
-|---------------|----------------------------------------|
-| Master Server | `master-server/conf/common.properties` |
-| Api Server    | `api-server/conf/common.properties`    |
-| Worker Server | `worker-server/conf/common.properties` |
-| Alert Server  | `alert-server/conf/common.properties`  |
+|    Service    |                          Configuration file                           |
+|---------------|-----------------------------------------------------------------------|
+| Master Server | `master-server/conf/common.properties`                                |
+| Api Server    | `api-server/conf/common.properties`, `api-server/conf/aws.yaml`       |
+| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` |
+| Alert Server  | `alert-server/conf/common.properties`                                 |
 
 The default configuration is as follows:
 
@@ -212,10 +214,6 @@ The default configuration is as follows:
 | data.basedir.path                             | /tmp/dolphinscheduler                            | local directory used to store temp files                                                                                                                                                                                                                                                                                                                                                                                             |
 | resource.storage.type                         | NONE                                             | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE                                                                                                                                                                                                                                                                                                                                                                                |
 | resource.upload.path                          | /dolphinscheduler                                | storage path of resource files                                                                                                                                                                                                                                                                                                                                                                                                       |
-| aws.access.key.id                             | minioadmin                                       | access key id of S3                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| aws.secret.access.key                         | minioadmin                                       | secret access key of S3                                                                                                                                                                                                                                                                                                                                                                                                              |
-| aws.region                                    | us-east-1                                        | region of S3                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| aws.s3.endpoint                               | http://minio:9000                                | endpoint of S3                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | hdfs.root.user                                | hdfs                                             | configure users with corresponding permissions if storage type is HDFS                                                                                                                                                                                                                                                                                                                                                               |
 | fs.defaultFS                                  | hdfs://mycluster:8020                            | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory                                                                                                                                                                                                           |
 | hadoop.security.authentication.startup.state  | false                                            | whether hadoop grant kerberos permission                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -330,7 +328,6 @@ Location: `worker-server/conf/application.yaml`
 | worker.registry-disconnect-strategy.max-waiting-time                        | 100s          | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
 | worker.task-execute-threads-full-policy                                     | REJECT        | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution                                                        |
 | worker.tenant-config.auto-create-tenant-enabled                             | true          | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.                                                                                           |
-| worker.tenant-config.distributed-tenant-enabled                             | false         | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants                                                                                                                                                                                                   |
 | worker.tenant-config.default-tenant-enabled                                 | false         | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.                                                                                                                                                                                                           |
 
 ### Alert Server related configuration

docs/docs/en/contribute/development-environment-setup.md

@@ -153,7 +153,7 @@ The browser access address [http://localhost:5173](http://localhost:5173) can lo
 
 #### zookeeper
 
-Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it.
+Download [ZooKeeper](https://zookeeper.apache.org/releases.html), and extract it.
 
 - Create directory `zkData` and `zkLog`
 - Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper

docs/docs/en/guide/installation/cluster.md

@@ -14,7 +14,7 @@ Configure all the configurations refer to [pseudo-cluster deployment](pseudo-clu
 
 ### Modify Configuration
 
-This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster .md), the following describes the parameters that must be modified:
+This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster.md), the following describes the parameters that must be modified:
 
 ```shell
 # ---------------------------------------------------------

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -2,7 +2,7 @@
 
 The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.
 
-If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
+If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
 
 ## Preparation
 

docs/docs/en/guide/installation/standalone.md

@@ -36,7 +36,7 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch
 
 ### Start or Stop Server
 
-The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
+The script `./bin/dolphinscheduler-daemon.sh` can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
 
 ```shell
 # Start Standalone Server

docs/docs/en/guide/metrics/metrics.md

@@ -91,6 +91,11 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
   - stop: the number of stopped workflow instances
   - failover: the number of workflow instance fail-overs
 
+### RPC Related Metrics
+
+- ds.rpc.client.sync.request.exception.count: (counter) the number of exceptions occurred in sync rpc requests
+- ds.rpc.client.sync.request.duration.time: (histogram) the time cost of sync rpc requests
+
 ### Master Server Metrics
 
 - ds.master.overload.count: (counter) the number of times the master overloaded

docs/docs/en/guide/parameter/global.md

@@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case,
 
 ### Save the workflow and set global parameters
 
-Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it.
+Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it.
 
 ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)
 

docs/docs/en/guide/parameter/project-parameter.md

@@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project.
 
 ### Define project-level parameters
 
-On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below:
+On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below:
 
 ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)
 

docs/docs/en/guide/parameter/startup-parameter.md

@@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config
 
 ## Usage
 
-Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters.
+Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters.
 
 ## Example
 

docs/docs/en/guide/resource/configuration.md

@@ -28,74 +28,37 @@ The configuration you may need to change:
 
 ## connect AWS S3
 
-if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
+if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following:
 
 config the following fields
 
 ```properties
-......
 
 resource.storage.type=S3
-
-......
-
-resource.aws.access.key.id=aws_access_key_id
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=aws_secret_access_key
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=us-west-2
-# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-resource.aws.s3.bucket.name=dolphinscheduler
-# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-resource.aws.s3.endpoint=
-
-......
 ```
 
-## Use HDFS or Remote Object Storage
+```yaml
+aws:
+    s3:
+        # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+        # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+        # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+        credentials.provider.type: AWSStaticCredentialsProvider
+        access.key.id: <access.key.id>
+        access.key.secret: <access.key.secret>
+        region: <region>
+        bucket.name: <bucket.name>
+        endpoint: <endpoint>
 
-After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`.
+```
+
+## connect OSS S3
+
+if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
+
+config the following fields
 
 ```properties
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# user data local directory path, please make sure the directory exists and have read write permissions
-data.basedir.path=/tmp/dolphinscheduler
-
-# resource view suffixs
-#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
-
-# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
-resource.storage.type=LOCAL
-# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
-resource.storage.upload.base.path=/tmp/dolphinscheduler
-
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=minioadmin
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=minioadmin
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=cn-north-1
-# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-resource.aws.s3.bucket.name=dolphinscheduler
-# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-resource.aws.s3.endpoint=http://localhost:9000
-
 # alibaba cloud access key id, required if you set resource.storage.type=OSS 
 resource.alibaba.cloud.access.key.id=<your-access-key-id>
 # alibaba cloud access key secret, required if you set resource.storage.type=OSS
@@ -107,89 +70,24 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
 
-# alibaba cloud access key id, required if you set resource.storage.type=OBS
+```
+
+## connect OBS S3
+
+if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
+
+config the following fields
+
+```properties
+# access key id, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.id=<your-access-key-id>
-# alibaba cloud access key secret, required if you set resource.storage.type=OBS
+# access key secret, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.secret=<your-access-key-secret>
 # oss bucket name, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
 
-# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
-resource.hdfs.root.user=hdfs
-# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
-
-# whether to startup kerberos
-hadoop.security.authentication.startup.state=false
-
-# java.security.krb5.conf path
-java.security.krb5.conf.path=/opt/krb5.conf
-
-# login user from keytab username
-login.user.keytab.username=hdfs-mycluster@ESZ.COM
-
-# login user from keytab path
-login.user.keytab.path=/opt/hdfs.headless.keytab
-
-# kerberos expire time, the unit is hour
-kerberos.expire.time=2
-
-
-# resourcemanager port, the default value is 8088 if not specified
-resource.manager.httpaddress.port=8088
-# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
-yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
-# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
-yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
-# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
-yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
-
-# datasource encryption enable
-datasource.encryption.enable=false
-
-# datasource encryption salt
-datasource.encryption.salt=!@#$%^&*
-
-# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
-# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
-# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
-data-quality.jar.dir=
-
-#data-quality.error.output.path=/tmp/data-quality-error-data
-
-# Network IP gets priority, default inner outer
-
-# Whether hive SQL is executed in the same session
-support.hive.oneSession=false
-
-# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
-sudo.enable=true
-
-# network interface preferred like eth0, default: empty
-#dolphin.scheduler.network.interface.preferred=
-
-# network IP gets priority, default: inner outer
-#dolphin.scheduler.network.priority.strategy=default
-
-# system env path
-#dolphinscheduler.env.path=dolphinscheduler_env.sh
-
-# development state
-development.state=false
-
-# rpc port
-alert.rpc.port=50052
-
-# set path of conda.sh
-conda.path=/opt/anaconda3/etc/profile.d/conda.sh
-
-# Task resource limit state
-task.resource.limit.state=false
-
-# way to collect applicationId: log(original regex match), aop
-appId.collect: log
 ```
 
 > **Note:**

docs/docs/en/guide/task/dms.md

@@ -73,14 +73,17 @@ Parameters of restarting the task by interface
 
 ## Environment to prepare
 
-Some AWS configuration is required, modify a field in file `common.properties`
+Some AWS configuration is required, modify a field in file `aws.yaml`
 
 ```yaml
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=<AWS REGION>
+dms:
+  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+  credentials.provider.type: AWSStaticCredentialsProvider
+  access.key.id: <access.key.id>
+  access.key.secret: <access.key.secret>
+  region: <region>
+  endpoint: <endpoint>
 ```
 

docs/docs/en/guide/task/sagemaker.md

@@ -35,14 +35,17 @@ The task plugin are shown as follows:
 
 ## Environment to prepare
 
-Some AWS configuration is required, modify a field in file `common.properties`
+Some AWS configuration is required, modify a field in file `aws.yaml`
 
 ```yaml
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=<AWS REGION>
+sagemaker:
+  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+  credentials.provider.type: AWSStaticCredentialsProvider
+  access.key.id: <access.key.id>
+  access.key.secret: <access.key.secret>
+  region: <region>
+  endpoint: <endpoint>
 ```
 

docs/docs/zh/architecture/configuration.md

@@ -331,7 +331,6 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 | worker.registry-disconnect-strategy.max-waiting-time                        | 100s      | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
 | worker.task-execute-threads-full-policy                                     | REJECT    | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务         |
 | worker.tenant-config.auto-create-tenant-enabled                             | true      | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。                                                                 |
-| worker.tenant-config.distributed-tenant-enabled                             | false     | 如果设置为true, auto-create-tenant-enabled 将会不起作用。                                                                                             |
 | worker.tenant-config.default-tenant-enabled                                 | false     | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。                                                                                               |
 
 ## Alert Server相关配置

docs/docs/zh/contribute/development-environment-setup.md

@@ -148,7 +148,7 @@ pnpm run dev
 
 #### zookeeper
 
-下载 [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3)，解压
+下载 [ZooKeeper](https://zookeeper.apache.org/releases.html)，解压
 
 * 在 ZooKeeper 的目录下新建 zkData、zkLog文件夹
 * 将 conf 目录下的 `zoo_sample.cfg` 文件，复制一份，重命名为 `zoo.cfg`，修改其中数据和日志的配置，如：

docs/docs/zh/contribute/join/contribute.md

@@ -32,7 +32,7 @@
 如果你想实现某个 Feature 或者修复某个 Bug。请参考以下内容：
 
 * 所有的 Bug 与新 Feature 建议使用 Issues Page 进行管理。
-* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加的回复内容中。
+* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加到回复内容中。
 * 最好在核心贡献者找到一个导师（指导者），导师会在设计与功能实现上给予即时的反馈。
 * 你应该新建一个分支来开始你的工作，分支的名字参考[参与贡献 Pull Request 需知](./pull-request.md)。比如，你想完成 feature 功能并提交了 Issue 111，那么你的 branch 名字应为 feature-111。 功能名称可与导师讨论后确定。
 * 完成后，发送一个 Pull Request 到 dolphinscheduler，提交过程具体请参考下面《[提交代码流程](./submit-code.md)》。

docs/docs/zh/guide/metrics/metrics.md

@@ -91,6 +91,11 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
   - stop：停止的工作流实例数量
   - failover：容错的工作流实例数量
 
+### RPC相关指标
+
+- ds.rpc.client.sync.request.exception.count: (counter) 同步rpc请求异常数
+- ds.rpc.client.sync.request.duration.time: (histogram) 同步rpc请求耗时
+
 ### Master Server指标
 
 - ds.master.overload.count: (counter) master过载次数

docs/docs/zh/guide/parameter/global.md

@@ -20,7 +20,7 @@
 
 ### 保存工作流，并设置全局参数
 
-全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示：
+全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示：
 
 ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)
 

docs/docs/zh/guide/parameter/project-parameter.md

@@ -8,7 +8,7 @@
 
 ### 定义项目级别参数
 
-在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示：
+在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示：
 
 ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)
 

docs/docs/zh/guide/parameter/startup-parameter.md

@@ -6,7 +6,7 @@
 
 ## 使用方式
 
-启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。
+启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。
 
 ## 任务样例
 

docs/docs/zh/guide/resource/configuration.md

@@ -26,77 +26,35 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
 
 ## 对接AWS S3
 
-如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
+如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下：
 
 配置以下字段
 
 ```properties
-......
 
 resource.storage.type=S3
-
-......
-
-resource.aws.access.key.id=aws_access_key_id
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=aws_secret_access_key
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=us-west-2
-# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-resource.aws.s3.bucket.name=dolphinscheduler
-# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-resource.aws.s3.endpoint=
-
-......
 ```
 
-## 对接分布式或远端对象存储
+```yaml
+aws:
+    s3:
+        # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+        # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+        # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+        credentials.provider.type: AWSStaticCredentialsProvider
+        access.key.id: <access.key.id>
+        access.key.secret: <access.key.secret>
+        region: <region>
+        bucket.name: <bucket.name>
+        endpoint: <endpoint>
 
-当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置：
+```
 
-### 配置 common.properties 文件
+## 对接阿里云 OSS
 
-在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
+如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
 
 ```properties
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# user data local directory path, please make sure the directory exists and have read write permissions
-data.basedir.path=/tmp/dolphinscheduler
-
-# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
-resource.storage.type=LOCAL
-
-# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
-# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
-resource.storage.upload.base.path=/tmp/dolphinscheduler
-
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=minioadmin
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=minioadmin
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=cn-north-1
-# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-resource.aws.s3.bucket.name=dolphinscheduler
-# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-resource.aws.s3.endpoint=http://localhost:9000
-
 # alibaba cloud access key id, required if you set resource.storage.type=OSS 
 resource.alibaba.cloud.access.key.id=<your-access-key-id>
 # alibaba cloud access key secret, required if you set resource.storage.type=OSS
@@ -108,87 +66,22 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
 
-# alibaba cloud access key id, required if you set resource.storage.type=OBS
+```
+
+## 对接华为云 OBS
+
+如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
+
+```properties
+# access key id, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.id=<your-access-key-id>
-# alibaba cloud access key secret, required if you set resource.storage.type=OBS
+# access key secret, required if you set resource.storage.type=OBS
 resource.huawei.cloud.access.key.secret=<your-access-key-secret>
 # oss bucket name, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OBS
 resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
 
-# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
-resource.hdfs.root.user=root
-# if resource.storage.type=S3, the value like: s3a://dolphinscheduler;
-# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-resource.hdfs.fs.defaultFS=hdfs://localhost:8020
-
-# whether to startup kerberos
-hadoop.security.authentication.startup.state=false
-
-# java.security.krb5.conf path
-java.security.krb5.conf.path=/opt/krb5.conf
-
-# login user from keytab username
-login.user.keytab.username=hdfs-mycluster@ESZ.COM
-
-# login user from keytab path
-login.user.keytab.path=/opt/hdfs.headless.keytab
-
-# kerberos expire time, the unit is hour
-kerberos.expire.time=2
-# resource view suffixs
-#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
-
-# resourcemanager port, the default value is 8088 if not specified
-resource.manager.httpaddress.port=8088
-# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
-yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
-# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value;
-# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
-yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s
-# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
-yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s
-
-# datasource encryption enable
-datasource.encryption.enable=false
-
-# datasource encryption salt
-datasource.encryption.salt=!@#$%^&*
-
-# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
-# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
-# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
-data-quality.jar.dir=
-
-#data-quality.error.output.path=/tmp/data-quality-error-data
-
-# Network IP gets priority, default inner outer
-
-# Whether hive SQL is executed in the same session
-support.hive.oneSession=false
-
-# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions;
-# if set false, executing user is the deploy user and doesn't need sudo permissions
-sudo.enable=true
-
-# network interface preferred like eth0, default: empty
-#dolphin.scheduler.network.interface.preferred=
-
-# network IP gets priority, default: inner outer
-#dolphin.scheduler.network.priority.strategy=default
-
-# system env path
-#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
-
-# development state
-development.state=false
-
-# rpc port
-alert.rpc.port=50052
-
-# way to collect applicationId: log(original regex match), aop
-appId.collect: log
 ```
 
 > **注意**：

docs/docs/zh/guide/task/appendix.md

@@ -8,7 +8,7 @@
 |----------|--------------------------------------------------------------------------------------------------------------------------------------|
 | 任务名称     | 任务的名称，同一个工作流定义中的节点名称不能重复。                                                                                                            |
 | 运行标志     | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。                                                                                                   |
-| 缓存执行     | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。                                              |
+| 缓存执行     | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。                                              |
 | 描述       | 当前节点的功能描述。                                                                                                                           |
 | 任务优先级    | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。                                                                                       |
 | Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。                                                                              |

docs/docs/zh/guide/task/dms.md

@@ -73,14 +73,17 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS
 
 ## 环境配置
 
-需要进行AWS的一些配置，修改`common.properties`中的以下配置信息
+需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
 
 ```yaml
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=<AWS REGION>
+dms:
+  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+  credentials.provider.type: AWSStaticCredentialsProvider
+  access.key.id: <access.key.id>
+  access.key.secret: <access.key.secret>
+  region: <region>
+  endpoint: <endpoint>
 ```
 

docs/docs/zh/guide/task/sagemaker.md

@@ -33,14 +33,17 @@ DolphinScheduler SageMaker 组件的功能:
 
 ## 环境配置
 
-需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息
+需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
 
 ```yaml
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=<AWS REGION>
+sagemaker:
+  # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+  # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+  # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+  credentials.provider.type: AWSStaticCredentialsProvider
+  access.key.id: <access.key.id>
+  access.key.secret: <access.key.secret>
+  region: <region>
+  endpoint: <endpoint>
 ```
 

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml

@@ -56,6 +56,13 @@
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
+            <includes>
+                <include>**/*.yaml</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
     </fileSets>
     <dependencySets>
         <dependencySet>

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh

@@ -24,6 +24,7 @@
 -XX:+PrintGCDetails
 -Xloggc:gc.log
 
+-XX:-OmitStackTraceInFastThrow
 -XX:+ExitOnOutOfMemoryError
 -XX:+HeapDumpOnOutOfMemoryError
 -XX:HeapDumpPath=dump.hprof

dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml

@@ -57,6 +57,13 @@
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
+            <includes>
+                <include>**/*.yaml</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
         <fileSet>
             <directory>${basedir}/../dolphinscheduler-ui/dist</directory>
             <outputDirectory>./ui</outputDirectory>

dolphinscheduler-api/src/main/bin/jvm_args_env.sh

@@ -24,6 +24,7 @@
 -XX:+PrintGCDetails
 -Xloggc:gc.log
 
+-XX:-OmitStackTraceInFastThrow
 -XX:+ExitOnOutOfMemoryError
 -XX:+HeapDumpOnOutOfMemoryError
 -XX:HeapDumpPath=dump.hprof

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java

@@ -126,7 +126,7 @@ public class OperatorUtils {
         return auditType.getAuditOperationType();
     }
 
-    public static long getObjectIdentityByParma(String[] paramNameArr, Map<String, Object> paramsMap) {
+    public static long getObjectIdentityByParam(String[] paramNameArr, Map<String, Object> paramsMap) {
         for (String name : paramNameArr) {
             if (paramsMap.get(name) instanceof String) {
                 String param = (String) paramsMap.get(name);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java

@@ -80,14 +80,14 @@ public abstract class BaseAuditOperator implements AuditOperator {
         }
 
         modifyRequestParams(paramNameArr, paramsMap, auditLogList);
-        setObjectByParma(paramNameArr, paramsMap, auditLogList);
+        setObjectByParam(paramNameArr, paramsMap, auditLogList);
 
         if (auditLogList.get(0).getModelId() == null) {
-            auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParma(paramNameArr, paramsMap));
+            auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParam(paramNameArr, paramsMap));
         }
     }
 
-    protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParam(String[] paramNameArr, Map<String, Object> paramsMap,
                                     List<AuditLog> auditLogList) {
 
         String name = paramNameArr[0];
@@ -97,7 +97,7 @@ public abstract class BaseAuditOperator implements AuditOperator {
             return;
         }
 
-        String objName = getObjectNameFromReturnIdentity(value);
+        String objName = getObjectNameFromIdentity(value);
 
         if (Strings.isNullOrEmpty(objName)) {
             auditLogList.get(0).setModelName(value.toString());
@@ -114,7 +114,7 @@ public abstract class BaseAuditOperator implements AuditOperator {
         auditLogList.get(0).setModelName(objName);
     }
 
-    protected void setObjectByParmaArr(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParamArr(String[] paramNameArr, Map<String, Object> paramsMap,
                                        List<AuditLog> auditLogList) {
 
         AuditLog auditLog = auditLogList.get(0);
@@ -127,7 +127,7 @@ public abstract class BaseAuditOperator implements AuditOperator {
             for (String identityString : identityArr) {
                 long identity = toLong(identityString);
 
-                String value = getObjectNameFromReturnIdentity(identity);
+                String value = getObjectNameFromIdentity(identity);
 
                 if (value == null) {
                     continue;
@@ -156,7 +156,7 @@ public abstract class BaseAuditOperator implements AuditOperator {
 
     protected void setObjectNameFromReturnIdentity(List<AuditLog> auditLogList) {
         auditLogList
-                .forEach(auditLog -> auditLog.setModelName(getObjectNameFromReturnIdentity(auditLog.getModelId())));
+                .forEach(auditLog -> auditLog.setModelName(getObjectNameFromIdentity(auditLog.getModelId())));
     }
 
     protected void modifyObjectFromReturnObject(String[] params, Map<String, Object> returnObjectMap,
@@ -180,7 +180,7 @@ public abstract class BaseAuditOperator implements AuditOperator {
         return NumberUtils.toLong(str.toString(), -1);
     }
 
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         return identity.toString();
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class AlertGroupAuditOperatorImpl extends BaseAuditOperator {
     private AlertGroupMapper alertGroupMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class AlertInstanceAuditOperatorImpl extends BaseAuditOperator {
     private AlertPluginInstanceMapper alertPluginInstanceMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class ClusterAuditOperatorImpl extends BaseAuditOperator {
     private ClusterMapper clusterMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class DatasourceAuditOperatorImpl extends BaseAuditOperator {
     private DataSourceMapper dataSourceMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class EnvironmentAuditOperatorImpl extends BaseAuditOperator {
     private EnvironmentMapper environmentMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class K8SNamespaceAuditOperatorImpl extends BaseAuditOperator {
     private K8sNamespaceMapper k8sNamespaceMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java

@@ -46,14 +46,14 @@ public class ProcessAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParam(String[] paramNameArr, Map<String, Object> paramsMap,
                                     List<AuditLog> auditLogList) {
         if (paramNameArr[0].equals(AuditLogConstants.CODES)
                 || paramNameArr[0].equals(AuditLogConstants.PROCESS_DEFINITION_CODES)
                 || paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) {
-            super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList);
+            super.setObjectByParamArr(paramNameArr, paramsMap, auditLogList);
         } else {
-            super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
+            super.setObjectByParam(paramNameArr, paramsMap, auditLogList);
         }
         if (paramsMap.containsKey(AuditLogConstants.VERSION)) {
             if (paramsMap.get(AuditLogConstants.VERSION) != null) {
@@ -65,7 +65,7 @@ public class ProcessAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java

@@ -48,17 +48,17 @@ public class ProcessInstanceAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParam(String[] paramNameArr, Map<String, Object> paramsMap,
                                     List<AuditLog> auditLogList) {
         if (paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) {
-            super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList);
+            super.setObjectByParamArr(paramNameArr, paramsMap, auditLogList);
         } else {
-            super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
+            super.setObjectByParam(paramNameArr, paramsMap, auditLogList);
         }
     }
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         int objId = NumberUtils.toInt(identity.toString(), -1);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java

@@ -34,7 +34,7 @@ public class ProjectAuditOperatorImpl extends BaseAuditOperator {
     private ProjectMapper projectMapper;
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java

@@ -37,7 +37,7 @@ public class ResourceAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParam(String[] paramNameArr, Map<String, Object> paramsMap,
                                     List<AuditLog> auditLogList) {
 
         Object objName = getFileNameFromParam(paramNameArr, paramsMap);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java

@@ -72,7 +72,7 @@ public class ScheduleAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java

@@ -46,17 +46,17 @@ public class TaskAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
+    protected void setObjectByParam(String[] paramNameArr, Map<String, Object> paramsMap,
                                     List<AuditLog> auditLogList) {
 
-        super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
+        super.setObjectByParam(paramNameArr, paramsMap, auditLogList);
         if (paramsMap.containsKey(AuditLogConstants.VERSION)) {
             auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString());
         }
     }
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class TaskGroupAuditOperatorImpl extends BaseAuditOperator {
     private TaskGroupMapper taskGroupMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class TaskInstancesAuditOperatorImpl extends BaseAuditOperator {
     private TaskInstanceMapper taskInstanceMapper;
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class TenantAuditOperatorImpl extends BaseAuditOperator {
     private TenantMapper tenantMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java

@@ -54,7 +54,7 @@ public class TokenAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java

@@ -33,7 +33,7 @@ public class UdfFunctionAuditOperatorImpl extends BaseAuditOperator {
     private UdfFuncMapper udfFuncMapper;
 
     @Override
-    protected String getObjectNameFromReturnIdentity(Object identity) {
+    protected String getObjectNameFromIdentity(Object identity) {
         int objId = NumberUtils.toInt(identity.toString(), -1);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class UserAuditOperatorImpl extends BaseAuditOperator {
     private UserMapper userMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java

@@ -48,7 +48,7 @@ public class WorkerGroupAuditOperatorImpl extends BaseAuditOperator {
     }
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java

@@ -31,7 +31,7 @@ public class YarnQueueAuditOperatorImpl extends BaseAuditOperator {
     private QueueMapper queueMapper;
 
     @Override
-    public String getObjectNameFromReturnIdentity(Object identity) {
+    public String getObjectNameFromIdentity(Object identity) {
         Long objId = toLong(identity);
         if (objId == -1) {
             return "";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java

@@ -44,6 +44,8 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
+import org.apache.dolphinscheduler.plugin.task.api.utils.PropertyUtils;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -163,10 +165,8 @@ public class ExecutorController extends BaseController {
         if (timeout == null) {
             timeout = Constants.MAX_TASK_TIMEOUT;
         }
-        Map<String, String> startParamMap = null;
-        if (startParams != null) {
-            startParamMap = JSONUtils.toMap(startParams);
-        }
+
+        List<Property> startParamList = PropertyUtils.startParamsTransformPropertyList(startParams);
 
         if (complementDependentMode == null) {
             complementDependentMode = ComplementDependentMode.OFF_MODE;
@@ -175,7 +175,7 @@ public class ExecutorController extends BaseController {
         Map<String, Object> result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode,
                 scheduleTime, execType, failureStrategy,
                 startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority,
-                workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun,
+                workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber, dryRun,
                 testFlag,
                 complementDependentMode, version, allLevelDependent, executionOrder);
         return returnDataList(result);
@@ -262,10 +262,7 @@ public class ExecutorController extends BaseController {
             timeout = Constants.MAX_TASK_TIMEOUT;
         }
 
-        Map<String, String> startParamMap = null;
-        if (startParams != null) {
-            startParamMap = JSONUtils.toMap(startParams);
-        }
+        List<Property> startParamList = PropertyUtils.startParamsTransformPropertyList(startParams);
 
         if (complementDependentMode == null) {
             log.debug("Parameter complementDependentMode set to {} due to null.", ComplementDependentMode.OFF_MODE);
@@ -283,7 +280,8 @@ public class ExecutorController extends BaseController {
             result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime,
                     execType, failureStrategy,
                     startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority,
-                    workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun,
+                    workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber,
+                    dryRun,
                     testFlag,
                     complementDependentMode, null, allLevelDependent, executionOrder);
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java

@@ -61,7 +61,8 @@ public class ProjectParameterController extends BaseController {
     @Operation(summary = "createProjectParameter", description = "CREATE_PROJECT_PARAMETER_NOTES")
     @Parameters({
             @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)),
-            @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class))
+            @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)),
+            @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class))
     })
     @PostMapping()
     @ResponseStatus(HttpStatus.CREATED)
@@ -69,9 +70,10 @@ public class ProjectParameterController extends BaseController {
     public Result createProjectParameter(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                          @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode,
                                          @RequestParam("projectParameterName") String projectParameterName,
-                                         @RequestParam(value = "projectParameterValue") String projectParameterValue) {
+                                         @RequestParam(value = "projectParameterValue") String projectParameterValue,
+                                         @RequestParam(value = "projectParameterDataType", defaultValue = "VARCHAR") String projectParameterDataType) {
         return projectParameterService.createProjectParameter(loginUser, projectCode, projectParameterName,
-                projectParameterValue);
+                projectParameterValue, projectParameterDataType);
     }
 
     @Operation(summary = "updateProjectParameter", description = "UPDATE_PROJECT_PARAMETER_NOTES")
@@ -79,6 +81,7 @@ public class ProjectParameterController extends BaseController {
             @Parameter(name = "code", description = "PROJECT_PARAMETER_CODE", schema = @Schema(implementation = long.class, example = "123456")),
             @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)),
             @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)),
+            @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class))
     })
     @PutMapping(value = "/{code}")
     @ResponseStatus(HttpStatus.OK)
@@ -87,9 +90,10 @@ public class ProjectParameterController extends BaseController {
                                          @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode,
                                          @PathVariable("code") Long code,
                                          @RequestParam("projectParameterName") String projectParameterName,
-                                         @RequestParam(value = "projectParameterValue") String projectParameterValue) {
+                                         @RequestParam(value = "projectParameterValue") String projectParameterValue,
+                                         @RequestParam(value = "projectParameterDataType") String projectParameterDataType) {
         return projectParameterService.updateProjectParameter(loginUser, projectCode, code, projectParameterName,
-                projectParameterValue);
+                projectParameterValue, projectParameterDataType);
     }
 
     @Operation(summary = "deleteProjectParametersByCode", description = "DELETE_PROJECT_PARAMETER_NOTES")
@@ -133,13 +137,14 @@ public class ProjectParameterController extends BaseController {
                                                   @Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                                   @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode,
                                                   @RequestParam(value = "searchVal", required = false) String searchVal,
+                                                  @RequestParam(value = "projectParameterDataType", required = false) String projectParameterDataType,
                                                   @RequestParam("pageNo") Integer pageNo,
                                                   @RequestParam("pageSize") Integer pageSize) {
 
         checkPageParams(pageNo, pageSize);
         searchVal = ParameterUtils.handleEscapes(searchVal);
         return projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo,
-                searchVal);
+                searchVal, projectParameterDataType);
     }
 
     @Operation(summary = "queryProjectParameterByCode", description = "QUERY_PROJECT_PARAMETER_NOTES")

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java

@@ -44,7 +44,7 @@ public class ApiExceptionHandler {
     @ExceptionHandler(Throwable.class)
     public Result<Object> exceptionHandler(Throwable e, HandlerMethod hm) {
         ApiException ce = hm.getMethodAnnotation(ApiException.class);
-        log.error("Meet en unknown exception: ", e);
+        log.error("Meet an unknown exception: ", e);
         if (ce == null) {
             return Result.errorWithArgs(Status.INTERNAL_SERVER_ERROR_ARGS, e.getMessage());
         }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@@ -30,7 +30,9 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -57,7 +59,7 @@ public interface ExecutorService {
      * @param environmentCode environment code
      * @param runMode run mode
      * @param timeout timeout
-     * @param startParams the global param values which pass to new process instance
+     * @param startParamList the global param values which pass to new process instance
      * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode
      * @param executionOrder the execution order when complementing data
      * @return execute process instance code
@@ -71,7 +73,7 @@ public interface ExecutorService {
                                             Priority processInstancePriority, String workerGroup, String tenantCode,
                                             Long environmentCode,
                                             Integer timeout,
-                                            Map<String, String> startParams, Integer expectedParallelismNumber,
+                                            List<Property> startParamList, Integer expectedParallelismNumber,
                                             int dryRun, int testFlag,
                                             ComplementDependentMode complementDependentMode, Integer version,
                                             boolean allLevelDependent, ExecutionOrder executionOrder);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java

@@ -23,17 +23,17 @@ import org.apache.dolphinscheduler.dao.entity.User;
 public interface ProjectParameterService {
 
     Result createProjectParameter(User loginUser, long projectCode, String projectParameterName,
-                                  String projectParameterValue);
+                                  String projectParameterValue, String projectParameterDataType);
 
     Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName,
-                                  String projectParameterValue);
+                                  String projectParameterValue, String projectParameterDataType);
 
     Result deleteProjectParametersByCode(User loginUser, long projectCode, long code);
 
     Result batchDeleteProjectParametersByCodes(User loginUser, long projectCode, String codes);
 
     Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo,
-                                           String searchVal);
+                                           String searchVal, String projectParameterDataType);
 
     Result queryProjectParameterByCode(User loginUser, long projectCode, long code);
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@@ -90,6 +90,7 @@ import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTrigg
 import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
 import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
 import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
 import org.apache.dolphinscheduler.service.command.CommandService;
 import org.apache.dolphinscheduler.service.cron.CronUtils;
@@ -204,7 +205,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
      * @param environmentCode           environment code
      * @param runMode                   run mode
      * @param timeout                   timeout
-     * @param startParams               the global param values which pass to new process instance
+     * @param startParamList               the global param values which pass to new process instance
      * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode
      * @param testFlag testFlag
      * @param executionOrder the execution order when complementing data
@@ -220,7 +221,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
                                                    Priority processInstancePriority, String workerGroup,
                                                    String tenantCode,
                                                    Long environmentCode, Integer timeout,
-                                                   Map<String, String> startParams, Integer expectedParallelismNumber,
+                                                   List<Property> startParamList, Integer expectedParallelismNumber,
                                                    int dryRun, int testFlag,
                                                    ComplementDependentMode complementDependentMode, Integer version,
                                                    boolean allLevelDependent, ExecutionOrder executionOrder) {
@@ -270,7 +271,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
                         startNodeList,
                         cronTime, warningType, loginUser.getId(), warningGroupId, runMode, processInstancePriority,
                         workerGroup, tenantCode,
-                        environmentCode, startParams, expectedParallelismNumber, dryRun, testFlag,
+                        environmentCode, startParamList, expectedParallelismNumber, dryRun, testFlag,
                         complementDependentMode, allLevelDependent, executionOrder);
 
         if (create > 0) {
@@ -732,7 +733,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
                               WarningType warningType, int executorId, Integer warningGroupId, RunMode runMode,
                               Priority processInstancePriority, String workerGroup, String tenantCode,
                               Long environmentCode,
-                              Map<String, String> startParams, Integer expectedParallelismNumber, int dryRun,
+                              List<Property> startParamList, Integer expectedParallelismNumber, int dryRun,
                               int testFlag, ComplementDependentMode complementDependentMode,
                               boolean allLevelDependent, ExecutionOrder executionOrder) {
 
@@ -761,8 +762,8 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
         if (warningType != null) {
             command.setWarningType(warningType);
         }
-        if (startParams != null && startParams.size() > 0) {
-            cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParams));
+        if (CollectionUtils.isNotEmpty(startParamList)) {
+            cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParamList));
         }
         command.setCommandParam(JSONUtils.toJsonString(cmdParam));
         command.setExecutorId(executorId);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java

@@ -68,7 +68,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
     @Override
     @Transactional
     public Result createProjectParameter(User loginUser, long projectCode, String projectParameterName,
-                                         String projectParameterValue) {
+                                         String projectParameterValue, String projectParameterDataType) {
         Result result = new Result();
 
         // check if user have write perm for project
@@ -97,6 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
                     .builder()
                     .paramName(projectParameterName)
                     .paramValue(projectParameterValue)
+                    .paramDataType(projectParameterDataType)
                     .code(CodeGenerateUtils.genCode())
                     .projectCode(projectCode)
                     .userId(loginUser.getId())
@@ -122,7 +123,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
 
     @Override
     public Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName,
-                                         String projectParameterValue) {
+                                         String projectParameterValue, String projectParameterDataType) {
         Result result = new Result();
 
         // check if user have write perm for project
@@ -155,6 +156,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
 
         projectParameter.setParamName(projectParameterName);
         projectParameter.setParamValue(projectParameterValue);
+        projectParameter.setParamDataType(projectParameterDataType);
         projectParameter.setUpdateTime(new Date());
         projectParameter.setOperator(loginUser.getId());
 
@@ -236,7 +238,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
 
     @Override
     public Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo,
-                                                  String searchVal) {
+                                                  String searchVal, String projectParameterDataType) {
         Result result = new Result();
 
         Project project = projectMapper.queryByCode(projectCode);
@@ -249,7 +251,8 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
         Page<ProjectParameter> page = new Page<>(pageNo, pageSize);
 
         IPage<ProjectParameter> iPage =
-                projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal);
+                projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal,
+                        projectParameterDataType);
 
         List<ProjectParameter> projectParameterList = iPage.getRecords();
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.api.service.impl;
 
-import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkFilePath;
 import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS;
 import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT;
 import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
@@ -1291,10 +1290,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
         if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) {
             return;
         }
-        // abnormal characters check
-        if (!checkFilePath(fullName)) {
-            throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH);
-        }
         // Avoid returning to the parent directory
         if (fullName.contains("../")) {
             throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

@@ -158,14 +158,4 @@ public class CheckUtils {
 
         return pattern.matcher(str).matches();
     }
-
-    /**
-     * regex FilePath check,only use a to z, A to Z, 0 to 9, and _./- 
-     *
-     * @param str     input string
-     * @return true if regex pattern is right, otherwise return false
-     */
-    public static boolean checkFilePath(String str) {
-        return regexChecks(str, Constants.REGEX_FILE_PATH);
-    }
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java

@@ -38,8 +38,13 @@ import org.apache.dolphinscheduler.common.enums.RunMode;
 import org.apache.dolphinscheduler.common.enums.TaskDependType;
 import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
+import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.junit.jupiter.api.Test;
@@ -75,7 +80,8 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest {
     final String tenantCode = "root";
     final Long environmentCode = 4L;
     final Integer timeout = 5;
-    final ImmutableMap<String, String> startParams = ImmutableMap.of("start", "params");
+    final List<Property> startParams =
+            Collections.singletonList(new Property("start", Direct.IN, DataType.VARCHAR, "params"));
     final Integer expectedParallelismNumber = 6;
     final int dryRun = 7;
     final int testFlag = 0;

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java

@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.enums.UserType;
 import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -48,8 +49,9 @@ public class ProjectParameterControllerTest {
         User loginUser = getGeneralUser();
 
         Mockito.when(projectParameterService.createProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.any(),
-                Mockito.any())).thenReturn(getSuccessResult());
-        Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value");
+                Mockito.any(), Mockito.any())).thenReturn(getSuccessResult());
+        Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value",
+                DataType.VARCHAR.name());
         Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode());
     }
 
@@ -58,8 +60,9 @@ public class ProjectParameterControllerTest {
         User loginUser = getGeneralUser();
 
         Mockito.when(projectParameterService.updateProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(),
-                Mockito.any(), Mockito.any())).thenReturn(getSuccessResult());
-        Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value");
+                Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult());
+        Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value",
+                DataType.LONG.name());
         Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode());
     }
 
@@ -88,8 +91,9 @@ public class ProjectParameterControllerTest {
         User loginUser = getGeneralUser();
 
         Mockito.when(projectParameterService.queryProjectParameterListPaging(Mockito.any(), Mockito.anyLong(),
-                Mockito.anyInt(), Mockito.anyInt(), Mockito.any())).thenReturn(getSuccessResult());
-        Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1", 1, 10);
+                Mockito.anyInt(), Mockito.anyInt(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult());
+        Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1",
+                DataType.VARCHAR.name(), 1, 10);
         Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode());
     }
 

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java

@@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.ProjectParameter;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
 import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
 
 import java.util.Collections;
 
@@ -76,7 +77,8 @@ public class ProjectParameterServiceTest {
         // PERMISSION DENIED
         when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class)))
                 .thenReturn(false);
-        Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value");
+        Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value",
+                DataType.VARCHAR.name());
         assertNull(result.getData());
         assertNull(result.getCode());
         assertNull(result.getMsg());
@@ -88,25 +90,29 @@ public class ProjectParameterServiceTest {
         try (MockedStatic<CodeGenerateUtils> ignored = Mockito.mockStatic(CodeGenerateUtils.class)) {
             when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class);
 
-            result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value");
+            result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value",
+                    DataType.VARCHAR.name());
             assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode());
         }
 
         // PROJECT_PARAMETER_ALREADY_EXISTS
         when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode));
         when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter());
-        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value");
+        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode());
 
         // INSERT DATA ERROR
         when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null);
         when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1);
-        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value");
+        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode());
 
         // SUCCESS
         when(projectParameterMapper.insert(Mockito.any())).thenReturn(1);
-        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value");
+        result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.SUCCESS.getCode(), result.getCode());
     }
 
@@ -117,7 +123,8 @@ public class ProjectParameterServiceTest {
         // NO PERMISSION
         when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class)))
                 .thenReturn(false);
-        Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value");
+        Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value",
+                DataType.VARCHAR.name());
         assertNull(result.getData());
         assertNull(result.getCode());
         assertNull(result.getMsg());
@@ -127,28 +134,33 @@ public class ProjectParameterServiceTest {
         when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class)))
                 .thenReturn(true);
         when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null);
-        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value");
+        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode());
 
         // PROJECT_PARAMETER_ALREADY_EXISTS
         when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter());
         when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter());
-        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value");
+        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode());
 
         // PROJECT_UPDATE_ERROR
         when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null);
         when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1);
-        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value");
+        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value",
+                DataType.VARCHAR.name());
         assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode());
 
         // SUCCESS
         when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1);
-        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value");
+        result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value",
+                DataType.LONG.name());
         assertEquals(Status.SUCCESS.getCode(), result.getCode());
         ProjectParameter projectParameter = (ProjectParameter) result.getData();
         assertNotNull(projectParameter.getOperator());
         assertNotNull(projectParameter.getUpdateTime());
+        assertEquals(DataType.LONG.name(), projectParameter.getParamDataType());
     }
 
     @Test
@@ -223,7 +235,8 @@ public class ProjectParameterServiceTest {
                         .thenReturn(false);
 
         Result result =
-                projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null);
+                projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null,
+                        DataType.VARCHAR.name());
         assertNull(result.getData());
         assertNull(result.getCode());
         assertNull(result.getMsg());
@@ -235,9 +248,10 @@ public class ProjectParameterServiceTest {
         Page<ProjectParameter> page = new Page<>(pageNo, pageSize);
         page.setRecords(Collections.singletonList(getProjectParameter()));
 
-        when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page);
+        when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any(), any()))
+                .thenReturn(page);
         result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo,
-                null);
+                null, null);
         assertEquals(Status.SUCCESS.getCode(), result.getCode());
     }
 

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java

@@ -92,24 +92,4 @@ public class CheckUtilsTest {
         Assertions.assertTrue(CheckUtils.checkPhone("17362537263"));
     }
 
-    /**
-     * check file path
-     */
-    @Test
-    public void testCheckFilePath() {
-        // true
-        Assertions.assertTrue(CheckUtils.checkFilePath("/"));
-        Assertions.assertTrue(CheckUtils.checkFilePath("xx/"));
-        Assertions.assertTrue(CheckUtils.checkFilePath("/xx"));
-        Assertions.assertTrue(CheckUtils.checkFilePath("14567134578654"));
-        Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/"));
-        Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/1531531..13513/153135.."));
-        // false
-        Assertions.assertFalse(CheckUtils.checkFilePath(null));
-        Assertions.assertFalse(CheckUtils.checkFilePath("file://xxx/ss"));
-        Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss;/dasd/123"));
-        Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss && /dasd/123"));
-        Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss || /dasd/123"));
-    }
-
 }

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.dolphinscheduler</groupId>
+        <artifactId>dolphinscheduler-authentication</artifactId>
+        <version>dev-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>dolphinscheduler-aws-authentication</artifactId>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-emr</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-sagemaker</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-dms</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>datasync</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import static org.apache.dolphinscheduler.authentication.aws.AwsConfigurationKeys.AWS_AUTHENTICATION_TYPE;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+
+@Slf4j
+@UtilityClass
+public class AWSCredentialsProviderFactor {
+
+    public static AWSCredentialsProvider credentialsProvider(Map<String, String> awsProperties) {
+        String awsAuthenticationType = awsProperties.getOrDefault(
+                AWS_AUTHENTICATION_TYPE, AWSCredentialsProviderType.STATIC_CREDENTIALS_PROVIDER.getName());
+        AWSCredentialsProviderType awsCredentialsProviderType =
+                AWSCredentialsProviderType.of(awsAuthenticationType).orElse(null);
+        if (awsCredentialsProviderType == null) {
+            throw new IllegalArgumentException(
+                    "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated");
+        }
+        switch (awsCredentialsProviderType) {
+            case STATIC_CREDENTIALS_PROVIDER:
+                return createAWSStaticCredentialsProvider(awsProperties);
+            case INSTANCE_PROFILE_CREDENTIALS_PROVIDER:
+                return createInstanceProfileCredentialsProvider();
+            default:
+                throw new IllegalArgumentException(
+                        "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated");
+        }
+
+    }
+
+    private static AWSCredentialsProvider createAWSStaticCredentialsProvider(Map<String, String> awsProperties) {
+        String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID);
+        String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET);
+        final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey);
+        AWSStaticCredentialsProvider awsStaticCredentialsProvider =
+                new AWSStaticCredentialsProvider(basicAWSCredentials);
+        log.info("AWSStaticCredentialsProvider created successfully");
+        return awsStaticCredentialsProvider;
+    }
+
+    private static AWSCredentialsProvider createInstanceProfileCredentialsProvider() {
+        InstanceProfileCredentialsProvider instanceProfileCredentialsProvider =
+                InstanceProfileCredentialsProvider.getInstance();
+        log.info("InstanceProfileCredentialsProvider created successfully");
+        return instanceProfileCredentialsProvider;
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import java.util.Optional;
+
+import lombok.Getter;
+
+@Getter
+public enum AWSCredentialsProviderType {
+
+    STATIC_CREDENTIALS_PROVIDER("AWSStaticCredentialsProvider"),
+    INSTANCE_PROFILE_CREDENTIALS_PROVIDER("InstanceProfileCredentialsProvider"),
+    ;
+
+    private final String name;
+
+    AWSCredentialsProviderType(String name) {
+        this.name = name;
+    }
+
+    public static Optional<AWSCredentialsProviderType> of(String name) {
+        if (name == null) {
+            return Optional.empty();
+        }
+        for (AWSCredentialsProviderType type : values()) {
+            if (type.getName().equalsIgnoreCase(name)) {
+                return Optional.of(type);
+            }
+        }
+        return Optional.empty();
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService;
+import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder;
+
+@UtilityClass
+public class AWSDatabaseMigrationServiceClientFactory {
+
+    public AWSDatabaseMigrationService createAWSDatabaseMigrationServiceClient(Map<String, String> awsProperties) {
+        AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
+        Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
+        String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
+
+        if (endpoint != null && !endpoint.isEmpty()) {
+            return AWSDatabaseMigrationServiceClientBuilder
+                    .standard()
+                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
+                    .withCredentials(awsCredentialsProvider)
+                    .build();
+        } else {
+            return AWSDatabaseMigrationServiceClientBuilder
+                    .standard()
+                    .withCredentials(awsCredentialsProvider)
+                    .withRegion(regions)
+                    .build();
+        }
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
+import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder;
+
+@UtilityClass
+public class AmazonElasticMapReduceClientFactory {
+
+    public AmazonElasticMapReduce createAmazonElasticMapReduceClient(Map<String, String> awsProperties) {
+        AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
+        Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
+        String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
+
+        if (endpoint != null && !endpoint.isEmpty()) {
+            return AmazonElasticMapReduceClientBuilder
+                    .standard()
+                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
+                    .withCredentials(awsCredentialsProvider)
+                    .build();
+        } else {
+            return AmazonElasticMapReduceClientBuilder
+                    .standard()
+                    .withCredentials(awsCredentialsProvider)
+                    .withRegion(regions)
+                    .build();
+        }
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+
+@UtilityClass
+public class AmazonS3ClientFactory {
+
+    public AmazonS3 createAmazonS3Client(Map<String, String> awsProperties) {
+        AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
+        Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
+        String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
+
+        if (endpoint != null && !endpoint.isEmpty()) {
+            return AmazonS3ClientBuilder
+                    .standard()
+                    .withPathStyleAccessEnabled(true)
+                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
+                    .withCredentials(awsCredentialsProvider)
+                    .build();
+        } else {
+            return AmazonS3ClientBuilder
+                    .standard()
+                    .withCredentials(awsCredentialsProvider)
+                    .withRegion(regions)
+                    .build();
+        }
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.sagemaker.AmazonSageMaker;
+import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder;
+
+@UtilityClass
+public class AmazonSageMakerClientFactory {
+
+    public AmazonSageMaker createAmazonSageMakerClient(Map<String, String> awsProperties) {
+        AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
+        Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
+        String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
+
+        if (endpoint != null && !endpoint.isEmpty()) {
+            return AmazonSageMakerClientBuilder
+                    .standard()
+                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
+                    .withCredentials(awsCredentialsProvider)
+                    .build();
+        } else {
+            return AmazonSageMakerClientBuilder
+                    .standard()
+                    .withCredentials(awsCredentialsProvider)
+                    .withRegion(regions)
+                    .build();
+        }
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+public class AwsConfigurationKeys {
+
+    public static final String AWS_AUTHENTICATION_TYPE = "credentials.provider.type";
+    public static final String AWS_REGION = "region";
+    public static final String AWS_ENDPOINT = "endpoint";
+
+    public static final String AWS_ACCESS_KEY_ID = "access.key.id";
+    public static final String AWS_SECRET = "access.key.secret";
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.authentication.aws;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.datasync.DataSyncClient;
+
+import java.util.Map;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class DataSyncClientFactory {
+
+    public DataSyncClient createDataSyncClient(Map<String, String> awsProperties) {
+        // todo: upgrade the version of aws sdk
+        String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID);
+        String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET);
+        final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey);
+        final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials);
+
+        // create a datasync client
+        return DataSyncClient.builder()
+                .region(Region.of(awsProperties.get(AwsConfigurationKeys.AWS_REGION)))
+                .credentialsProvider(awsCredentialsProvider)
+                .build();
+    }
+
+}

dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+aws:
+  s3:
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    credentials.provider.type: AWSStaticCredentialsProvider
+    access.key.id: accessKey123
+    access.key.secret: secretKey123
+    region: us-east-1
+    bucket.name: dolphinscheduler
+    endpoint: http://s3:9000
+  emr:
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    credentials.provider.type: AWSStaticCredentialsProvider
+    access.key.id: minioadmin
+    access.key.secret: minioadmin
+    region: cn-north-1
+    endpoint: http://localhost:9000
+  sagemaker:
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    credentials.provider.type: AWSStaticCredentialsProvider
+    access.key.id: minioadmin
+    access.key.secret: minioadmin
+    region: cn-north-1
+    endpoint: http://localhost:9000
+  dms:
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    credentials.provider.type: AWSStaticCredentialsProvider
+    access.key.id: minioadmin
+    access.key.secret: minioadmin
+    region: cn-north-1
+    endpoint: http://localhost:9000
+  datasync:
+    # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
+    # AWSStaticCredentialsProvider: use the access key and secret key to authenticate
+    # InstanceProfileCredentialsProvider: use the IAM role to authenticate
+    credentials.provider.type: AWSStaticCredentialsProvider
+    access.key.id: minioadmin
+    access.key.secret: minioadmin
+    region: cn-north-1
+    endpoint: http://localhost:9000
+

dolphinscheduler-authentication/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.dolphinscheduler</groupId>
+        <artifactId>dolphinscheduler</artifactId>
+        <version>dev-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>dolphinscheduler-authentication</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>dolphinscheduler-aws-authentication</module>
+    </modules>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.dolphinscheduler</groupId>
+                <artifactId>dolphinscheduler-bom</artifactId>
+                <version>${project.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+</project>

dolphinscheduler-common/pom.xml

@@ -41,6 +41,10 @@
     </dependencyManagement>
     <dependencies>
         <!-- dolphinscheduler -->
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-aws-authentication</artifactId>
+        </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

@@ -36,6 +36,7 @@ public final class Constants {
     public static final String COMMON_PROPERTIES_PATH = "/common.properties";
 
     public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
+    public static final String AWS_YAML_PATH = "/aws.yaml";
 
     public static final String FORMAT_SS = "%s%s";
     public static final String FORMAT_S_S = "%s/%s";
@@ -130,8 +131,7 @@ public final class Constants {
      */
     public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type";
 
-    public static final String AWS_S3_BUCKET_NAME = "resource.aws.s3.bucket.name";
-    public static final String AWS_END_POINT = "resource.aws.s3.endpoint";
+    public static final String AWS_S3_BUCKET_NAME = "aws.s3.bucket.name";
 
     public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name";
     public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint";
@@ -249,11 +249,6 @@ public final class Constants {
      */
     public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,39}$");
 
-    /**
-     * file path regex 
-     */
-    public static final Pattern REGEX_FILE_PATH = Pattern.compile("^[a-zA-Z0-9_./-]+$");
-
     /**
      * read permission
      */
@@ -704,19 +699,8 @@ public final class Constants {
 
     public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint";
 
-    /**
-     * remote logging for S3
-     */
-    public static final String REMOTE_LOGGING_S3_ACCESS_KEY_ID = "remote.logging.s3.access.key.id";
-
-    public static final String REMOTE_LOGGING_S3_ACCESS_KEY_SECRET = "remote.logging.s3.access.key.secret";
-
     public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name";
 
-    public static final String REMOTE_LOGGING_S3_ENDPOINT = "remote.logging.s3.endpoint";
-
-    public static final String REMOTE_LOGGING_S3_REGION = "remote.logging.s3.region";
-
     /**
      * remote logging for GCS
      */

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/TenantConstants.java

@@ -21,5 +21,5 @@ public class TenantConstants {
 
     public static final String DEFAULT_TENANT_CODE = "default";
 
-    public static final String BOOTSTRAPT_SYSTEM_USER = System.getProperty("user.name");
+    public static final String BOOTSTRAP_SYSTEM_USER = System.getProperty("user.name");
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java

@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.common.log.remote;
 
+import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory;
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 
@@ -26,41 +27,25 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.Map;
 
 import lombok.extern.slf4j.Slf4j;
 
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.regions.Regions;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
 
 @Slf4j
 public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
 
-    private String accessKeyId;
+    private final String bucketName;
 
-    private String accessKeySecret;
-
-    private String region;
-
-    private String bucketName;
-
-    private String endPoint;
-
-    private AmazonS3 s3Client;
+    private final AmazonS3 s3Client;
 
     private static S3RemoteLogHandler instance;
 
     private S3RemoteLogHandler() {
-        accessKeyId = readAccessKeyID();
-        accessKeySecret = readAccessKeySecret();
-        region = readRegion();
         bucketName = readBucketName();
-        endPoint = readEndPoint();
         s3Client = buildS3Client();
         checkBucketNameExists(bucketName);
     }
@@ -74,23 +59,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
     }
 
     protected AmazonS3 buildS3Client() {
-        if (StringUtils.isNotEmpty(endPoint)) {
-            return AmazonS3ClientBuilder
-                    .standard()
-                    .withPathStyleAccessEnabled(true)
-                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
-                            endPoint, Regions.fromName(region).getName()))
-                    .withCredentials(
-                            new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
-                    .build();
-        } else {
-            return AmazonS3ClientBuilder
-                    .standard()
-                    .withCredentials(
-                            new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
-                    .withRegion(Regions.fromName(region))
-                    .build();
-        }
+        Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.s3.", "");
+        return AmazonS3ClientFactory.createAmazonS3Client(awsProperties);
     }
 
     @Override
@@ -131,24 +101,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
         }
     }
 
-    protected String readAccessKeyID() {
-        return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_ID);
-    }
-
-    protected String readAccessKeySecret() {
-        return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_SECRET);
-    }
-
-    protected String readRegion() {
-        return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_REGION);
-    }
-
     protected String readBucketName() {
-        return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_BUCKET_NAME);
-    }
-
-    protected String readEndPoint() {
-        return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ENDPOINT);
+        return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME);
     }
 
     public void checkBucketNameExists(String bucketName) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.common.utils;
 
+import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH;
 import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
 import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
 
@@ -42,7 +43,7 @@ public class PropertyUtils {
     private final ImmutablePriorityPropertyDelegate propertyDelegate =
             new ImmutablePriorityPropertyDelegate(
                     new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
-                    new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
+                    new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH));
 
     public static String getString(String key) {
         return propertyDelegate.get(key.trim());
@@ -106,6 +107,19 @@ public class PropertyUtils {
         return matchedProperties;
     }
 
+    /**
+     * Get all properties with specified prefix, like: fs., will replace the prefix with newPrefix
+     */
+    public static Map<String, String> getByPrefix(String prefix, String newPrefix) {
+        Map<String, String> matchedProperties = new HashMap<>();
+        for (String propName : propertyDelegate.getPropertyKeys()) {
+            if (propName.startsWith(prefix)) {
+                matchedProperties.put(propName.replace(prefix, newPrefix), propertyDelegate.get(propName));
+            }
+        }
+        return matchedProperties;
+    }
+
     public static <T> Set<T> getSet(String key, Function<String, Set<T>> transformFunction, Set<T> defaultValue) {
         return propertyDelegate.get(key, transformFunction, defaultValue);
     }

dolphinscheduler-common/src/main/resources/common.properties

@@ -39,17 +39,6 @@ resource.azure.tenant.id=minioadmin
 # The query interval
 resource.query.interval=10000
 
-# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.access.key.id=minioadmin
-# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.secret.access.key=minioadmin
-# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-resource.aws.region=cn-north-1
-# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
-resource.aws.s3.bucket.name=dolphinscheduler
-# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-resource.aws.s3.endpoint=http://localhost:9000
-
 # alibaba cloud access key id, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.access.key.id=<your-access-key-id>
 # alibaba cloud access key secret, required if you set resource.storage.type=OSS
@@ -188,16 +177,7 @@ remote.logging.oss.access.key.secret=<access.key.secret>
 remote.logging.oss.bucket.name=<bucket.name>
 # oss endpoint, required if you set remote.logging.target=OSS
 remote.logging.oss.endpoint=<endpoint>
-# s3 access key id, required if you set remote.logging.target=S3
-remote.logging.s3.access.key.id=<access.key.id>
-# s3 access key secret, required if you set remote.logging.target=S3
-remote.logging.s3.access.key.secret=<access.key.secret>
-# s3 bucket name, required if you set remote.logging.target=S3
-remote.logging.s3.bucket.name=<bucket.name>
-# s3 endpoint, required if you set remote.logging.target=S3
-remote.logging.s3.endpoint=<endpoint>
-# s3 region, required if you set remote.logging.target=S3
-remote.logging.s3.region=<region>
+
 # the location of the google cloud credential, required if you set remote.logging.target=GCS
 remote.logging.google.cloud.storage.credential=/path/to/credential
 # gcs bucket name, required if you set remote.logging.target=GCS

dolphinscheduler-common/src/main/resources/remote-logging.yaml

@@ -34,28 +34,16 @@ remote-logging:
     bucket.name: <bucket.name>
     # oss endpoint, required if you set remote-logging.target=OSS
     endpoint: <endpoint>
-  # required if you set remote-logging.target=S3
-  s3:
-    # s3 access key id, required if you set remote-logging.target=S3
-    access.key.id: <access.key.id>
-    # s3 access key secret, required if you set remote-logging.target=S3
-    access.key.secret: <access.key.secret>
-      # s3 bucket name, required if you set remote-logging.target=S3
-    bucket.name: <bucket.name>
-      # s3 endpoint, required if you set remote-logging.target=S3
-    endpoint: <endpoint>
-      # s3 region, required if you set remote-logging.target=S3
-    region: <region>
   google.cloud.storage:
     # the location of the google cloud credential, required if you set remote-logging.target=GCS
     credential: /path/to/credential
-      # gcs bucket name, required if you set remote-logging.target=GCS
+    # gcs bucket name, required if you set remote-logging.target=GCS
     bucket.name: <your-bucket>
   abs:
     # abs account name, required if you set resource.storage.type=ABS
     account.name: <your-account-name>
-      # abs account key, required if you set resource.storage.type=ABS
+    # abs account key, required if you set resource.storage.type=ABS
     account.key: <your-account-key>
-      # abs container name, required if you set resource.storage.type=ABS
+    # abs container name, required if you set resource.storage.type=ABS
     container.name: <your-container-name>
 

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java

@@ -17,12 +17,15 @@
 
 package org.apache.dolphinscheduler.common.utils;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import org.apache.dolphinscheduler.common.constants.Constants;
 
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -48,4 +51,14 @@ public class PropertyUtilsTest {
         }, Sets.newHashSet("docker0"));
         Assertions.assertEquals(Sets.newHashSet("docker0"), networkInterface);
     }
+
+    @Test
+    void getByPrefix() {
+        Map<String, String> awsProperties = PropertyUtils.getByPrefix("resource.aws.", "");
+        assertThat(awsProperties).containsEntry("access.key.id", "minioadmin");
+        assertThat(awsProperties).containsEntry("secret.access.key", "minioadmin");
+        assertThat(awsProperties).containsEntry("region", "cn-north-1");
+        assertThat(awsProperties).containsEntry("s3.bucket.name", "dolphinscheduler");
+        assertThat(awsProperties).containsEntry("endpoint", "http://localhost:9000");
+    }
 }