diff --git a/deploy/docker/.env b/deploy/docker/.env index d65a873b72..ad703258aa 100755 --- a/deploy/docker/.env +++ b/deploy/docker/.env @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. # -HUB=apache -TAG=3.2.2 +HUB=ghcr.io/apache/dolphinscheduler +TAG=latest TZ=Asia/Shanghai DATABASE=postgresql diff --git a/deploy/kubernetes/dolphinscheduler/Chart.yaml b/deploy/kubernetes/dolphinscheduler/Chart.yaml index 096d7b84c9..bb886ea8c5 100644 --- a/deploy/kubernetes/dolphinscheduler/Chart.yaml +++ b/deploy/kubernetes/dolphinscheduler/Chart.yaml @@ -35,7 +35,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 3.2.2 +version: 3.1.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 209e4e3942..ba533b6e47 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -120,12 +120,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false | | conf.common."alert.rpc.port" | int | `50052` | rpc port | | conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop | -| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | | -| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required | -| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required | -| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. | -| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn | -| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required | | conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh | | conf.common."data-quality.jar.dir" | string | `nil` | data quality option | | conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions | @@ -144,6 +138,11 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS | | conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS | | conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS | +| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required | +| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required | +| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. | +| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn | +| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required | | conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS | | conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS | | conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS | @@ -159,7 +158,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.common."yarn.application.status.address" | string | `"http://ds1:%s/ws/v1/cluster/apps/%s"` | if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname | | conf.common."yarn.job.history.status.address" | string | `"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"` | job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) | | conf.common."yarn.resourcemanager.ha.rm.ids" | string | `"192.168.xx.xx,192.168.xx.xx"` | if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty | -| datasource.profile | string | `"postgresql"` | The profile of datasource | | externalDatabase.database | string | `"dolphinscheduler"` | The database of external database | | externalDatabase.driverClassName | string | `"org.postgresql.Driver"` | The driverClassName of external database | | externalDatabase.enabled | bool | `false` | If exists external database, and set postgresql.enable value to false. external database will be used, otherwise Dolphinscheduler's internal database will be used. | diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 368e0b290f..71287b1f10 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -51,6 +51,7 @@ Create a default common labels. {{- define "dolphinscheduler.common.labels" -}} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} {{- end -}} {{/* diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index fd89c1ed15..268c78bc4e 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -33,13 +33,15 @@ data: banner: charset: UTF-8 datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler - username: root - password: root - hikari: - connection-test-query: select 1 - pool-name: DolphinScheduler + profile: postgresql + config: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler # Mybatis-plus configuration, you don't need to change it mybatis-plus: @@ -100,16 +102,4 @@ data: metrics: enabled: true - - # Override by profile - --- - spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler - username: root - password: root {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml index f115490c31..7570f9b010 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml @@ -54,13 +54,15 @@ data: messages: basename: i18n/messages datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler - username: root - password: root - hikari: - connection-test-query: select 1 - pool-name: DolphinScheduler + profile: postgresql + config: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler quartz: auto-startup: false job-store-type: jdbc @@ -237,21 +239,6 @@ data: application-name: "" # Doplhinscheduler login url redirect-url: "" - - # Override by profile - --- - spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler - username: root - password: root - quartz: - properties: - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml index 0a2c8bf2a4..07e8352f0d 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -33,13 +33,15 @@ data: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler - username: root - password: root - hikari: - connection-test-query: select 1 - pool-name: DolphinScheduler + profile: postgresql + config: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler quartz: job-store-type: jdbc jdbc: @@ -155,19 +157,4 @@ data: metrics: enabled: true - - # Override by profile - --- - spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler - username: root - password: root - quartz: - properties: - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml index 8c1d515ca7..c9af1c00a5 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml @@ -37,7 +37,7 @@ data: {{- range $key, $value := index .Values.conf "common" }} {{- if and $.Values.minio.enabled }} {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }} - {{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }} + {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }} {{- end }} {{ $key }}={{ $value }} {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 6a5560b3d4..84f28f484b 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -131,5 +131,5 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-configs - name: alert-config-volume configMap: - name: {{ include "dolphinscheduler.fullname" . }}-alert + name: { { include "dolphinscheduler.fullname" . } }-alert {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index f36d2442ae..0b359f15c8 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -141,7 +141,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-configs - name: api-config-volume configMap: - name: {{ include "dolphinscheduler.fullname" . }}-api + name: { { include "dolphinscheduler.fullname" . } }-api {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} {{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 1d9d517d0e..a93621b7f2 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -127,7 +127,7 @@ spec: {{- end }} - name: master-config-volume configMap: - name: {{ include "dolphinscheduler.fullname" . }}-master + name: { { include "dolphinscheduler.fullname" . } }-master {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} - name: config-volume configMap: diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 8d55be6f70..83ce8947dd 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -144,7 +144,7 @@ spec: {{- end }} - name: worker-config-volume configMap: - name: {{ include "dolphinscheduler.fullname" . }}-worker + name: { { include "dolphinscheduler.fullname" . } }-worker - name: config-volume configMap: name: {{ include "dolphinscheduler.fullname" . }}-configs diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 63799b2a65..7a04ff5604 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -33,7 +33,7 @@ image: # -- Docker image repository for the DolphinScheduler registry: apache # -- Docker image version for the DolphinScheduler - tag: 3.2.2 + tag: latest # -- Image pull policy. Options: Always, Never, IfNotPresent pullPolicy: "IfNotPresent" # -- Specify a imagePullSecrets @@ -49,10 +49,6 @@ image: # -- tools image tools: dolphinscheduler-tools -datasource: - # -- The profile of datasource - profile: postgresql - postgresql: # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL enabled: true @@ -250,25 +246,20 @@ conf: # -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path: /dolphinscheduler - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - aws.credentials.provider.type: AWSStaticCredentialsProvider + # -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required + resource.aws.access.key.id: minioadmin - # -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required - aws.s3.access.key.id: minioadmin + # -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required + resource.aws.secret.access.key: minioadmin - # -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required - aws.s3.access.key.secret: minioadmin - - # -- The AWS Region to use. if resource.storage.type=S3, This configuration is required - aws.s3.region: ca-central-1 + # -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required + resource.aws.region: ca-central-1 # -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. - aws.s3.bucket.name: dolphinscheduler + resource.aws.s3.bucket.name: dolphinscheduler # -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn - aws.s3.endpoint: http://minio:9000 + resource.aws.s3.endpoint: http://minio:9000 # -- alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id: diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index ce02146794..323e8b21d9 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -25,19 +25,19 @@ export default { children: [ { title: 'Introduction', - link: '/en-us/docs/3.2.2/user_doc/about/introduction.html', + link: '/en-us/docs/dev/user_doc/about/introduction.html', }, { title: 'Features', - link: '/en-us/docs/3.2.2/user_doc/about/features.html', + link: '/en-us/docs/dev/user_doc/about/features.html', }, { title: 'Hardware Environment', - link: '/en-us/docs/3.2.2/user_doc/about/hardware.html', + link: '/en-us/docs/dev/user_doc/about/hardware.html', }, { title: 'Glossary', - link: '/en-us/docs/3.2.2/user_doc/about/glossary.html', + link: '/en-us/docs/dev/user_doc/about/glossary.html', } ], }, @@ -46,11 +46,11 @@ export default { children: [ { title: 'Quick Start', - link: '/en-us/docs/3.2.2/user_doc/guide/start/quick-start.html', + link: '/en-us/docs/dev/user_doc/guide/start/quick-start.html', }, { title: 'Docker Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/start/docker.html', + link: '/en-us/docs/dev/user_doc/guide/start/docker.html', }, ], }, @@ -59,30 +59,30 @@ export default { children: [ { title: 'Application Homepage', - link: '/en-us/docs/3.2.2/user_doc/guide/homepage.html', + link: '/en-us/docs/dev/user_doc/guide/homepage.html', }, { title: 'Project', children: [ { title: 'Project List', - link: '/en-us/docs/3.2.2/user_doc/guide/project/project-list.html', + link: '/en-us/docs/dev/user_doc/guide/project/project-list.html', }, { title: 'Workflow Definition', - link: '/en-us/docs/3.2.2/user_doc/guide/project/workflow-definition.html', + link: '/en-us/docs/dev/user_doc/guide/project/workflow-definition.html', }, { title: 'Workflow Instance', - link: '/en-us/docs/3.2.2/user_doc/guide/project/workflow-instance.html', + link: '/en-us/docs/dev/user_doc/guide/project/workflow-instance.html', }, { title: 'Task Instance', - link: '/en-us/docs/3.2.2/user_doc/guide/project/task-instance.html', + link: '/en-us/docs/dev/user_doc/guide/project/task-instance.html', }, { title: 'Task Definition', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/task-definition.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/task-definition.html', }, ] }, @@ -91,147 +91,147 @@ export default { children: [ { title: 'Appendix', - link: '/en-us/docs/3.2.2/user_doc/guide/task/appendix.html', + link: '/en-us/docs/dev/user_doc/guide/task/appendix.html', }, { title: 'Shell', - link: '/en-us/docs/3.2.2/user_doc/guide/task/shell.html', + link: '/en-us/docs/dev/user_doc/guide/task/shell.html', }, { title: 'SubProcess', - link: '/en-us/docs/3.2.2/user_doc/guide/task/sub-process.html', + link: '/en-us/docs/dev/user_doc/guide/task/sub-process.html', }, { title: 'Dynamic', - link: '/en-us/docs/3.2.2/user_doc/guide/task/dynamic.html', + link: '/en-us/docs/dev/user_doc/guide/task/dynamic.html', }, { title: 'Dependent', - link: '/en-us/docs/3.2.2/user_doc/guide/task/dependent.html', + link: '/en-us/docs/dev/user_doc/guide/task/dependent.html', }, { title: 'Stored Procedure', - link: '/en-us/docs/3.2.2/user_doc/guide/task/stored-procedure.html', + link: '/en-us/docs/dev/user_doc/guide/task/stored-procedure.html', }, { title: 'SQL', - link: '/en-us/docs/3.2.2/user_doc/guide/task/sql.html', + link: '/en-us/docs/dev/user_doc/guide/task/sql.html', }, { title: 'Spark', - link: '/en-us/docs/3.2.2/user_doc/guide/task/spark.html', + link: '/en-us/docs/dev/user_doc/guide/task/spark.html', }, { title: 'MapReduce', - link: '/en-us/docs/3.2.2/user_doc/guide/task/map-reduce.html', + link: '/en-us/docs/dev/user_doc/guide/task/map-reduce.html', }, { title: 'Python', - link: '/en-us/docs/3.2.2/user_doc/guide/task/python.html', + link: '/en-us/docs/dev/user_doc/guide/task/python.html', }, { title: 'Flink', - link: '/en-us/docs/3.2.2/user_doc/guide/task/flink.html', + link: '/en-us/docs/dev/user_doc/guide/task/flink.html', }, { title: 'HTTP', - link: '/en-us/docs/3.2.2/user_doc/guide/task/http.html', + link: '/en-us/docs/dev/user_doc/guide/task/http.html', }, { title: 'DataX', - link: '/en-us/docs/3.2.2/user_doc/guide/task/datax.html', + link: '/en-us/docs/dev/user_doc/guide/task/datax.html', }, { title: 'Sqoop', - link: '/en-us/docs/3.2.2/user_doc/guide/task/sqoop.html', + link: '/en-us/docs/dev/user_doc/guide/task/sqoop.html', }, { title: 'Pigeon', - link: '/en-us/docs/3.2.2/user_doc/guide/task/pigeon.html', + link: '/en-us/docs/dev/user_doc/guide/task/pigeon.html', }, { title: 'Conditions', - link: '/en-us/docs/3.2.2/user_doc/guide/task/conditions.html', + link: '/en-us/docs/dev/user_doc/guide/task/conditions.html', }, { title: 'Switch', - link: '/en-us/docs/3.2.2/user_doc/guide/task/switch.html', + link: '/en-us/docs/dev/user_doc/guide/task/switch.html', }, { title: 'SeaTunnel', - link: '/en-us/docs/3.2.2/user_doc/guide/task/seatunnel.html', + link: '/en-us/docs/dev/user_doc/guide/task/seatunnel.html', }, { title: 'Amazon EMR', - link: '/en-us/docs/3.2.2/user_doc/guide/task/emr.html', + link: '/en-us/docs/dev/user_doc/guide/task/emr.html', }, { title: 'Apache Zeppelin', - link: '/en-us/docs/3.2.2/user_doc/guide/task/zeppelin.html', + link: '/en-us/docs/dev/user_doc/guide/task/zeppelin.html', }, { title: 'Jupyter', - link: '/en-us/docs/3.2.2/user_doc/guide/task/jupyter.html', + link: '/en-us/docs/dev/user_doc/guide/task/jupyter.html', }, { title: 'Hive CLI', - link: '/en-us/docs/3.2.2/user_doc/guide/task/hive-cli.html', + link: '/en-us/docs/dev/user_doc/guide/task/hive-cli.html', }, { title: 'Kubernetes', - link: '/en-us/docs/3.2.2/user_doc/guide/task/kubernetes.html', + link: '/en-us/docs/dev/user_doc/guide/task/kubernetes.html', }, { title: 'MLflow', - link: '/en-us/docs/3.2.2/user_doc/guide/task/mlflow.html', + link: '/en-us/docs/dev/user_doc/guide/task/mlflow.html', }, { title: 'Openmldb', - link: '/en-us/docs/3.2.2/user_doc/guide/task/openmldb.html', + link: '/en-us/docs/dev/user_doc/guide/task/openmldb.html', }, { title: 'DVC', - link: '/en-us/docs/3.2.2/user_doc/guide/task/dvc.html', + link: '/en-us/docs/dev/user_doc/guide/task/dvc.html', }, { title: 'Dinky', - link: '/en-us/docs/3.2.2/user_doc/guide/task/dinky.html', + link: '/en-us/docs/dev/user_doc/guide/task/dinky.html', }, { title: 'Java', - link: '/en-us/docs/3.2.2/user_doc/guide/task/java.html', + link: '/en-us/docs/dev/user_doc/guide/task/java.html', }, { title: 'SageMaker', - link: '/en-us/docs/3.2.2/user_doc/guide/task/sagemaker.html', + link: '/en-us/docs/dev/user_doc/guide/task/sagemaker.html', }, { title: 'ChunJun', - link: '/en-us/docs/3.2.2/user_doc/guide/task/chunjun.html', + link: '/en-us/docs/dev/user_doc/guide/task/chunjun.html', }, { title: 'Pytorch', - link: '/en-us/docs/3.2.2/user_doc/guide/task/pytorch.html', + link: '/en-us/docs/dev/user_doc/guide/task/pytorch.html', }, { title: 'Amazon DMS', - link: '/en-us/docs/3.2.2/user_doc/guide/task/dms.html', + link: '/en-us/docs/dev/user_doc/guide/task/dms.html', }, { title: 'AWS Datasync', - link: '/en-us/docs/3.2.2/user_doc/guide/task/datasync.html', + link: '/en-us/docs/dev/user_doc/guide/task/datasync.html', }, { title: 'Kubeflow', - link: '/en-us/docs/3.2.2/user_doc/guide/task/kubeflow.html', + link: '/en-us/docs/dev/user_doc/guide/task/kubeflow.html', }, { title: 'Apache Linkis', - link: '/en-us/docs/3.2.2/user_doc/guide/task/linkis.html', + link: '/en-us/docs/dev/user_doc/guide/task/linkis.html', }, { title: 'SSH', - link: '/en-us/docs/3.2.2/user_doc/guide/task/ssh.html', + link: '/en-us/docs/dev/user_doc/guide/task/ssh.html', }, ], }, @@ -240,39 +240,39 @@ export default { children: [ { title: 'Built-in Parameter', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/built-in.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/built-in.html', }, { title: 'Global Parameter', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/global.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/global.html', }, { title: 'Local Parameter', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/local.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/local.html', }, { title: 'Project-level Parameter', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/project-parameter.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/project-parameter.html', }, { title: 'Parameter Context', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/context.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/context.html', }, { title: 'Parameter Priority', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/priority.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/priority.html', }, { title: 'File Parameter', - link: '/en-us/docs/3.2.2/user_doc/guide/parameter/file-parameter.html', + link: '/en-us/docs/dev/user_doc/guide/parameter/file-parameter.html', }, { title: 'Vertica', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/vertica.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/vertica.html', }, { title: 'Remote Shell', - link: '/en-us/docs/3.2.2/user_doc/guide/task/remoteshell.html', + link: '/en-us/docs/dev/user_doc/guide/task/remoteshell.html', }, ], }, @@ -281,91 +281,91 @@ export default { children: [ { title: 'MySQL', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/mysql.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/mysql.html', }, { title: 'PostgreSQL', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/postgresql.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/postgresql.html', }, { title: 'HIVE', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/hive.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/hive.html', }, { title: 'SNOWFLAKE', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/snowflake.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/snowflake.html', }, { title: 'KYUUBI', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/kyuubi.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/kyuubi.html', }, { title: 'Spark', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/spark.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/spark.html', }, { title: 'Presto', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/presto.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/presto.html', }, { title: 'SQL SERVER', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/sqlserver.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/sqlserver.html', }, { title: 'Amazon Redshift', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/redshift.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/redshift.html', }, { title: 'ClickHouse', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/clickhouse.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/clickhouse.html', }, { title: 'IBM DB2', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/db2.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/db2.html', }, { title: 'Oracle', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/oracle.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/oracle.html', }, { title: 'Amazon Athena', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/athena.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/athena.html', }, { title: 'Azure SQL Database', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/azure-sql.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/azure-sql.html', }, { title: 'Trino', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/trino.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/trino.html', }, { title: 'StarRocks', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/starrocks.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/starrocks.html', }, { title: 'DAMENG', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/dameng.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/dameng.html', }, { title: 'OceanBase', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/oceanbase.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/oceanbase.html', }, { title: 'SSH', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/ssh.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/ssh.html', }, { title: 'Databend', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/databend.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/databend.html', }, { title: 'HANA', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/hana.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/hana.html', }, { title: 'doris', - link: '/en-us/docs/3.2.2/user_doc/guide/datasource/doris.html', + link: '/en-us/docs/dev/user_doc/guide/datasource/doris.html', } ], }, @@ -374,46 +374,46 @@ export default { children: [ { title: 'Alert Component User Guide ', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/alert_plugin_user_guide.html', + link: '/en-us/docs/dev/user_doc/guide/alert/alert_plugin_user_guide.html', }, { title: 'Email', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/email.html', + link: '/en-us/docs/dev/user_doc/guide/alert/email.html', }, { title: 'Telegram', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/telegram.html', + link: '/en-us/docs/dev/user_doc/guide/alert/telegram.html', }, { title: 'Ding Talk', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/dingtalk.html', + link: '/en-us/docs/dev/user_doc/guide/alert/dingtalk.html', }, { title: 'Enterprise Wechat', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/enterprise-wechat.html', + link: '/en-us/docs/dev/user_doc/guide/alert/enterprise-wechat.html', }, { title: 'Enterprise Webexteams', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/enterprise-webexteams.html', + link: '/en-us/docs/dev/user_doc/guide/alert/enterprise-webexteams.html', }, { title: 'Script', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/script.html', + link: '/en-us/docs/dev/user_doc/guide/alert/script.html', }, { title: 'Http', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/http.html', + link: '/en-us/docs/dev/user_doc/guide/alert/http.html', }, { title: 'Feishu', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/feishu.html', + link: '/en-us/docs/dev/user_doc/guide/alert/feishu.html', }, { title: 'Slack', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/slack.html', + link: '/en-us/docs/dev/user_doc/guide/alert/slack.html', },{ title: 'Aliyun Voice', - link: '/en-us/docs/3.2.2/user_doc/guide/alert/aliyun-voice.html', + link: '/en-us/docs/dev/user_doc/guide/alert/aliyun-voice.html', }, ], }, @@ -422,40 +422,40 @@ export default { children: [ { title: 'Introduction', - link: '/en-us/docs/3.2.2/user_doc/guide/resource/intro.html' + link: '/en-us/docs/dev/user_doc/guide/resource/intro.html' }, { title: 'Configuration', - link: '/en-us/docs/3.2.2/user_doc/guide/resource/configuration.html' + link: '/en-us/docs/dev/user_doc/guide/resource/configuration.html' }, { title: 'File Manage', - link: '/en-us/docs/3.2.2/user_doc/guide/resource/file-manage.html' + link: '/en-us/docs/dev/user_doc/guide/resource/file-manage.html' }, { title: 'UDF Manage', - link: '/en-us/docs/3.2.2/user_doc/guide/resource/udf-manage.html' + link: '/en-us/docs/dev/user_doc/guide/resource/udf-manage.html' }, { title: 'Task Group Manage', - link: '/en-us/docs/3.2.2/user_doc/guide/resource/task-group.html' + link: '/en-us/docs/dev/user_doc/guide/resource/task-group.html' }, ], }, { title: 'Monitor', - link: '/en-us/docs/3.2.2/user_doc/guide/monitor.html', + link: '/en-us/docs/dev/user_doc/guide/monitor.html', }, { title: 'Security', children: [ { title: 'Security (Authorization System)', - link: '/en-us/docs/3.2.2/user_doc/guide/security/security.html', + link: '/en-us/docs/dev/user_doc/guide/security/security.html', }, { title: 'Authentication Type', - link: '/en-us/docs/3.2.2/user_doc/guide/security/authentication-type.html', + link: '/en-us/docs/dev/user_doc/guide/security/authentication-type.html', } ], }, @@ -465,38 +465,38 @@ export default { children: [ { title: 'Metrics', - link: '/en-us/docs/3.2.2/user_doc/guide/metrics/metrics.html', + link: '/en-us/docs/dev/user_doc/guide/metrics/metrics.html', } ], }, { title: 'Data Quality', - link: '/en-us/docs/3.2.2/user_doc/guide/data-quality.html', + link: '/en-us/docs/dev/user_doc/guide/data-quality.html', }, { title: 'Remote Logging', - link: '/en-us/docs/3.2.2/user_doc/guide/remote-logging.html', + link: '/en-us/docs/dev/user_doc/guide/remote-logging.html', }, { title: 'Upgrade', children: [ { title: 'Incompatible', - link: '/en-us/docs/3.2.2/user_doc/guide/upgrade/incompatible.html', + link: '/en-us/docs/dev/user_doc/guide/upgrade/incompatible.html', }, { title: 'Upgrade', - link: '/en-us/docs/3.2.2/user_doc/guide/upgrade/upgrade.html', + link: '/en-us/docs/dev/user_doc/guide/upgrade/upgrade.html', }, ], }, { title: 'Expansion and Reduction', - link: '/en-us/docs/3.2.2/user_doc/guide/expansion-reduction.html', + link: '/en-us/docs/dev/user_doc/guide/expansion-reduction.html', }, { title: 'Demo', - link: '/en-us/docs/3.2.2/user_doc/guide/demo.html', + link: '/en-us/docs/dev/user_doc/guide/demo.html', }, ], }, @@ -505,26 +505,26 @@ export default { children: [ { title: 'Standalone Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/installation/standalone.html', + link: '/en-us/docs/dev/user_doc/guide/installation/standalone.html', }, { title: 'Pseudo Cluster Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/installation/pseudo-cluster.html', + link: '/en-us/docs/dev/user_doc/guide/installation/pseudo-cluster.html', }, { title: 'Cluster Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/installation/cluster.html', + link: '/en-us/docs/dev/user_doc/guide/installation/cluster.html', }, { title: 'Kubernetes Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/installation/kubernetes.html', + link: '/en-us/docs/dev/user_doc/guide/installation/kubernetes.html', }, { title: 'integration', children: [ { title: 'Rainbond Deployment', - link: '/en-us/docs/3.2.2/user_doc/guide/integration/rainbond.html', + link: '/en-us/docs/dev/user_doc/guide/integration/rainbond.html', }, ], }, @@ -535,27 +535,27 @@ export default { children: [ { title: 'Architecture Design', - link: '/en-us/docs/3.2.2/user_doc/architecture/design.html', + link: '/en-us/docs/dev/user_doc/architecture/design.html', }, { title: 'Metadata', - link: '/en-us/docs/3.2.2/user_doc/architecture/metadata.html', + link: '/en-us/docs/dev/user_doc/architecture/metadata.html', }, { title: 'Configuration File', - link: '/en-us/docs/3.2.2/user_doc/architecture/configuration.html', + link: '/en-us/docs/dev/user_doc/architecture/configuration.html', }, { title: 'Task Structure', - link: '/en-us/docs/3.2.2/user_doc/architecture/task-structure.html', + link: '/en-us/docs/dev/user_doc/architecture/task-structure.html', }, { title: 'Load Balance', - link: '/en-us/docs/3.2.2/user_doc/architecture/load-balance.html', + link: '/en-us/docs/dev/user_doc/architecture/load-balance.html', }, { title: 'Cache', - link: '/en-us/docs/3.2.2/user_doc/architecture/cache.html', + link: '/en-us/docs/dev/user_doc/architecture/cache.html', }, ], }, @@ -564,15 +564,15 @@ export default { children: [ { title: 'Open API', - link: '/en-us/docs/3.2.2/user_doc/guide/api/open-api.html', + link: '/en-us/docs/dev/user_doc/guide/api/open-api.html', }, { title: 'PyDolphinScheduler', - link: '/en-us/docs/3.2.2/user_doc/guide/api/pydolphinscheduler.html', + link: '/en-us/docs/dev/user_doc/guide/api/pydolphinscheduler.html', }, { title: 'Health Check', - link: '/en-us/docs/3.2.2/user_doc/guide/api/healthcheck.html', + link: '/en-us/docs/dev/user_doc/guide/api/healthcheck.html', }, ], }, @@ -584,69 +584,69 @@ export default { children: [ { title: 'Security Report', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/security.html', + link: '/en-us/docs/dev/user_doc/contribute/join/security.html', }, { title: 'How to Become a Committer', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/become-a-committer.html', + link: '/en-us/docs/dev/user_doc/contribute/join/become-a-committer.html', }, { title: 'Subscribe Mailing Lists', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/subscribe.html', + link: '/en-us/docs/dev/user_doc/contribute/join/subscribe.html', }, { title: 'Participate in Contributing', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/contribute.html', + link: '/en-us/docs/dev/user_doc/contribute/join/contribute.html', }, { title: 'Code of Conduct', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/code-conduct.html', + link: '/en-us/docs/dev/user_doc/contribute/join/code-conduct.html', }, { title: 'Review Issue or Pull Requests', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/review.html', + link: '/en-us/docs/dev/user_doc/contribute/join/review.html', }, { title: 'E2E Contribution Guide', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/e2e-guide.html', + link: '/en-us/docs/dev/user_doc/contribute/join/e2e-guide.html', }, { title: 'Submit Code', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/submit-code.html', + link: '/en-us/docs/dev/user_doc/contribute/join/submit-code.html', }, { title: 'License Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/DS-License.html', + link: '/en-us/docs/dev/user_doc/contribute/join/DS-License.html', }, { title: 'Document Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/document.html', + link: '/en-us/docs/dev/user_doc/contribute/join/document.html', }, { title: 'Issue Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/issue.html', + link: '/en-us/docs/dev/user_doc/contribute/join/issue.html', }, { title: 'Pull Request Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/pull-request.html', + link: '/en-us/docs/dev/user_doc/contribute/join/pull-request.html', }, { title: 'Commit Message Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/commit-message.html', + link: '/en-us/docs/dev/user_doc/contribute/join/commit-message.html', }, { title: 'Micro BenchMark Notice', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/microbench.html', + link: '/en-us/docs/dev/user_doc/contribute/join/microbench.html', }, { title: 'Unit Test Writing Guide', - link: '/en-us/docs/3.2.2/user_doc/contribute/join/unit-test.html', + link: '/en-us/docs/dev/user_doc/contribute/join/unit-test.html', }, ], }, { - title: '3.2.2elopment Environment Setup', - link: '/en-us/docs/3.2.2/user_doc/contribute/3.2.2elopment-environment-setup.html', + title: 'Development Environment Setup', + link: '/en-us/docs/dev/user_doc/contribute/development-environment-setup.html', }, { title: 'Design Document', @@ -657,27 +657,27 @@ export default { // children: [ { title: 'Architecture Design', - link: '/en-us/docs/3.2.2/user_doc/contribute/architecture-design.html', + link: '/en-us/docs/dev/user_doc/contribute/architecture-design.html', }, { title: 'Alert SPI', - link: '/en-us/docs/3.2.2/user_doc/contribute/backend/spi/alert.html', + link: '/en-us/docs/dev/user_doc/contribute/backend/spi/alert.html', }, { title: 'Registry SPI', - link: '/en-us/docs/3.2.2/user_doc/contribute/backend/spi/registry.html', + link: '/en-us/docs/dev/user_doc/contribute/backend/spi/registry.html', }, { title: 'Task SPI', - link: '/en-us/docs/3.2.2/user_doc/contribute/backend/spi/task.html', + link: '/en-us/docs/dev/user_doc/contribute/backend/spi/task.html', }, { title: 'Datasource SPI', - link: '/en-us/docs/3.2.2/user_doc/contribute/backend/spi/datasource.html', + link: '/en-us/docs/dev/user_doc/contribute/backend/spi/datasource.html', }, { title: 'Mechanism Design', - link: '/en-us/docs/3.2.2/user_doc/contribute/backend/mechanism/overview.html', + link: '/en-us/docs/dev/user_doc/contribute/backend/mechanism/overview.html', }, ], }, @@ -685,34 +685,34 @@ export default { title: 'Guidelines', children: [ { - title: 'Frontend 3.2.2elopment', - link: '/en-us/docs/3.2.2/user_doc/contribute/frontend-3.2.2elopment.html', + title: 'Frontend Development', + link: '/en-us/docs/dev/user_doc/contribute/frontend-development.html', }, { title: 'API Standard', - link: '/en-us/docs/3.2.2/user_doc/contribute/api-standard.html', + link: '/en-us/docs/dev/user_doc/contribute/api-standard.html', }, { title: 'E2E Automation Test', - link: '/en-us/docs/3.2.2/user_doc/contribute/e2e-test.html', + link: '/en-us/docs/dev/user_doc/contribute/e2e-test.html', }, { title: 'API Automation Test', - link: '/en-us/docs/3.2.2/user_doc/contribute/api-test.html', + link: '/en-us/docs/dev/user_doc/contribute/api-test.html', }, { title: 'Log Specification', - link: '/en-us/docs/3.2.2/user_doc/contribute/log-specification.html', + link: '/en-us/docs/dev/user_doc/contribute/log-specification.html', }, ], }, { title: 'Releasing', - link: '/en-us/docs/3.2.2/user_doc/contribute/release.html', + link: '/en-us/docs/dev/user_doc/contribute/release.html', }, { title: 'Questions & Communications', - link: '/en-us/docs/3.2.2/user_doc/contribute/have-questions.html', + link: '/en-us/docs/dev/user_doc/contribute/have-questions.html', }, ], }, @@ -721,7 +721,7 @@ export default { children: [ { title: 'DSIP', - link: '/en-us/docs/3.2.2/user_doc/DSIP.html', + link: '/en-us/docs/dev/user_doc/DSIP.html', }, ], }, @@ -730,11 +730,11 @@ export default { children: [ { title: 'General Setting', - link: '/en-us/docs/3.2.2/user_doc/guide/howto/general-setting.html', + link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html', }, { title: 'Datasource Setting', - link: '/en-us/docs/3.2.2/user_doc/guide/howto/datasource-setting.html', + link: '/en-us/docs/dev/user_doc/guide/howto/datasource-setting.html', }, { title: 'Others', @@ -761,19 +761,19 @@ export default { children: [ { title: '简介', - link: '/zh-cn/docs/3.2.2/user_doc/about/introduction.html', + link: '/zh-cn/docs/dev/user_doc/about/introduction.html', }, { title: '特性', - link: '/zh-cn/docs/3.2.2/user_doc/about/features.html', + link: '/zh-cn/docs/dev/user_doc/about/features.html', }, { title: '建议配置', - link: '/zh-cn/docs/3.2.2/user_doc/about/hardware.html', + link: '/zh-cn/docs/dev/user_doc/about/hardware.html', }, { title: '名词解释', - link: '/zh-cn/docs/3.2.2/user_doc/about/glossary.html', + link: '/zh-cn/docs/dev/user_doc/about/glossary.html', }, ], }, @@ -782,11 +782,11 @@ export default { children: [ { title: '快速上手', - link: '/zh-cn/docs/3.2.2/user_doc/guide/start/quick-start.html', + link: '/zh-cn/docs/dev/user_doc/guide/start/quick-start.html', }, { title: 'Docker部署(Docker)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/start/docker.html', + link: '/zh-cn/docs/dev/user_doc/guide/start/docker.html', }, ], }, @@ -795,30 +795,30 @@ export default { children: [ { title: '应用首页', - link: '/zh-cn/docs/3.2.2/user_doc/guide/homepage.html', + link: '/zh-cn/docs/dev/user_doc/guide/homepage.html', }, { title: '项目管理', children: [ { title: '项目列表', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/project-list.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/project-list.html', }, { title: '工作流定义', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/workflow-definition.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/workflow-definition.html', }, { title: '工作流实例', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/workflow-instance.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/workflow-instance.html', }, { title: '任务实例', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/task-instance.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/task-instance.html', }, { title: '任务定义', - link: '/zh-cn/docs/3.2.2/user_doc/guide/project/task-definition.html', + link: '/zh-cn/docs/dev/user_doc/guide/project/task-definition.html', }, ] }, @@ -827,155 +827,155 @@ export default { children: [ { title: 'Appendix', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/appendix.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/appendix.html', }, { title: 'Shell', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/shell.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/shell.html', }, { title: 'SubProcess', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/sub-process.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/sub-process.html', }, { title: 'Dynamic', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/dynamic.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/dynamic.html', }, { title: 'Dependent', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/dependent.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/dependent.html', }, { title: 'Stored Procedure', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/stored-procedure.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/stored-procedure.html', }, { title: 'SQL', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/sql.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/sql.html', }, { title: 'Spark', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/spark.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/spark.html', }, { title: 'MapReduce', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/map-reduce.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/map-reduce.html', }, { title: 'Python', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/python.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/python.html', }, { title: 'Flink', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/flink.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/flink.html', }, { title: 'HTTP', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/http.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/http.html', }, { title: 'DataX', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/datax.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/datax.html', }, { title: 'Sqoop', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/sqoop.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/sqoop.html', }, { title: 'Pigeon', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/pigeon.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/pigeon.html', }, { title: 'Conditions', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/conditions.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/conditions.html', }, { title: 'Switch', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/switch.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/switch.html', }, { title: 'SeaTunnel', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/seatunnel.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/seatunnel.html', }, { title: 'Amazon EMR', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/emr.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/emr.html', }, { title: 'Apache Zeppelin', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/zeppelin.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/zeppelin.html', }, { title: 'Jupyter', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/jupyter.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/jupyter.html', }, { title: 'Hive CLI', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/hive-cli.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/hive-cli.html', }, { title: 'Kubernetes', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/kubernetes.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/kubernetes.html', }, { title: 'MLflow', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/mlflow.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/mlflow.html', }, { title: 'Openmldb', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/openmldb.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/openmldb.html', }, { title: 'DVC', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/dvc.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/dvc.html', }, { title: 'Dinky', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/dinky.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/dinky.html', }, { title: 'Java', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/java.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/java.html', }, { title: 'SageMaker', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/sagemaker.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/sagemaker.html', }, { title: 'ChunJun', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/chunjun.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/chunjun.html', }, { title: 'Pytorch', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/pytorch.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/pytorch.html', }, { title: 'Amazon DMS', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/dms.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/dms.html', }, { title: 'AWS Datasync', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/datasync.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/datasync.html', }, { title: 'Kubeflow', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/kubeflow.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/kubeflow.html', }, { title: 'Apache Linkis', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/linkis.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/linkis.html', }, { title: 'SSH', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/ssh.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/ssh.html', }, { title: 'Vertica', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/vertica.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/vertica.html', }, { title: 'Remote Shell', - link: '/zh-cn/docs/3.2.2/user_doc/guide/task/remoteshell.html', + link: '/zh-cn/docs/dev/user_doc/guide/task/remoteshell.html', }, ], }, @@ -984,31 +984,31 @@ export default { children: [ { title: '内置参数', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/built-in.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/built-in.html', }, { title: '全局参数', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/global.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/global.html', }, { title: '本地参数', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/local.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/local.html', }, { title: '项目级别参数', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/project-parameter.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/project-parameter.html', }, { title: '参数传递', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/context.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/context.html', }, { title: '参数优先级', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/priority.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/priority.html', }, { title: '文件参数传递', - link: '/zh-cn/docs/3.2.2/user_doc/guide/parameter/file-parameter.html', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/file-parameter.html', }, ], }, @@ -1017,75 +1017,75 @@ export default { children: [ { title: 'MySQL', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/mysql.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/mysql.html', }, { title: 'PostgreSQL', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/postgresql.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/postgresql.html', }, { title: 'HIVE', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/hive.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/hive.html', }, { title: 'SNOWFLAKE', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/snowflake.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/snowflake.html', }, { title: 'KYUUBI', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/kyuubi.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/kyuubi.html', }, { title: 'Spark', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/spark.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/spark.html', }, { title: 'Amazon Athena', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/athena.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/athena.html', }, { title: 'Azure SQL Database', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/azure-sql.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/azure-sql.html', }, { title: 'Redshift', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/redshift.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/redshift.html', }, { title: 'Oracle', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/oracle.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/oracle.html', }, { title: 'Trino', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/trino.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/trino.html', }, { title: 'StarRocks', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/starrocks.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/starrocks.html', }, { title: 'DAMENG', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/dameng.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/dameng.html', }, { title: 'OceanBase', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/oceanbase.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/oceanbase.html', }, { title: 'SSH', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/ssh.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/ssh.html', }, { title: 'Databend', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/databend.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/databend.html', }, { title: 'HANA', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/hana.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/hana.html', }, { title: 'Doris', - link: '/zh-cn/docs/3.2.2/user_doc/guide/datasource/doris.html', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/doris.html', } ], }, @@ -1094,47 +1094,47 @@ export default { children: [ { title: '告警组件向导', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/alert_plugin_user_guide.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/alert_plugin_user_guide.html', }, { title: 'Email', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/email.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/email.html', }, { title: 'Telegram', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/telegram.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/telegram.html', }, { title: '钉钉告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/dingtalk.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/dingtalk.html', }, { title: '企业微信', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/enterprise-wechat.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/enterprise-wechat.html', }, { title: 'Webexteams', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/enterprise-webexteams.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/enterprise-webexteams.html', }, { title: '脚本告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/script.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/script.html', }, { title: 'Http告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/http.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/http.html', }, { title: '飞书告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/feishu.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/feishu.html', }, { title: 'Slack告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/slack.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/slack.html', }, { title: '阿里云语音告警', - link: '/zh-cn/docs/3.2.2/user_doc/guide/alert/aliyun-voice.html', + link: '/zh-cn/docs/dev/user_doc/guide/alert/aliyun-voice.html', }, ], }, @@ -1143,40 +1143,40 @@ export default { children: [ { title: '简介', - link: '/zh-cn/docs/3.2.2/user_doc/guide/resource/intro.html' + link: '/zh-cn/docs/dev/user_doc/guide/resource/intro.html' }, { title: '配置详情', - link: '/zh-cn/docs/3.2.2/user_doc/guide/resource/configuration.html' + link: '/zh-cn/docs/dev/user_doc/guide/resource/configuration.html' }, { title: '文件管理', - link: '/zh-cn/docs/3.2.2/user_doc/guide/resource/file-manage.html' + link: '/zh-cn/docs/dev/user_doc/guide/resource/file-manage.html' }, { title: 'UDF 管理', - link: '/zh-cn/docs/3.2.2/user_doc/guide/resource/udf-manage.html' + link: '/zh-cn/docs/dev/user_doc/guide/resource/udf-manage.html' }, { title: '任务组管理', - link: '/zh-cn/docs/3.2.2/user_doc/guide/resource/task-group.html' + link: '/zh-cn/docs/dev/user_doc/guide/resource/task-group.html' }, ], }, { title: '监控中心', - link: '/zh-cn/docs/3.2.2/user_doc/guide/monitor.html', + link: '/zh-cn/docs/dev/user_doc/guide/monitor.html', }, { title: '安全中心', children: [ { title: '安全中心（权限系统）', - link: '/zh-cn/docs/3.2.2/user_doc/guide/security/security.html', + link: '/zh-cn/docs/dev/user_doc/guide/security/security.html', }, { title: '认证方式', - link: '/zh-cn/docs/3.2.2/user_doc/guide/security/authentication-type.html', + link: '/zh-cn/docs/dev/user_doc/guide/security/authentication-type.html', } ], }, @@ -1185,38 +1185,38 @@ export default { children: [ { title: '指标', - link: '/zh-cn/docs/3.2.2/user_doc/guide/metrics/metrics.html', + link: '/zh-cn/docs/dev/user_doc/guide/metrics/metrics.html', } ], }, { title: '数据质量', - link: '/zh-cn/docs/3.2.2/user_doc/guide/data-quality.html', + link: '/zh-cn/docs/dev/user_doc/guide/data-quality.html', }, { title: '远程日志存储', - link: '/zh-cn/docs/3.2.2/user_doc/guide/remote-logging.html', + link: '/zh-cn/docs/dev/user_doc/guide/remote-logging.html', }, { title: '升级', children: [ { title: '不向前兼容的更新', - link: '/zh-cn/docs/3.2.2/user_doc/guide/upgrade/incompatible.html', + link: '/zh-cn/docs/dev/user_doc/guide/upgrade/incompatible.html', }, { title: '升级步骤', - link: '/zh-cn/docs/3.2.2/user_doc/guide/upgrade/upgrade.html', + link: '/zh-cn/docs/dev/user_doc/guide/upgrade/upgrade.html', }, ], }, { title: '扩/缩容', - link: '/zh-cn/docs/3.2.2/user_doc/guide/expansion-reduction.html', + link: '/zh-cn/docs/dev/user_doc/guide/expansion-reduction.html', }, { title: 'Demo', - link: '/zh-cn/docs/3.2.2/user_doc/guide/demo.html', + link: '/zh-cn/docs/dev/user_doc/guide/demo.html', }, ], }, @@ -1225,26 +1225,26 @@ export default { children: [ { title: '单机部署(Standalone)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/installation/standalone.html', + link: '/zh-cn/docs/dev/user_doc/guide/installation/standalone.html', }, { title: '伪集群部署(Pseudo-Cluster)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/installation/pseudo-cluster.html', + link: '/zh-cn/docs/dev/user_doc/guide/installation/pseudo-cluster.html', }, { title: '集群部署(Cluster)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/installation/cluster.html', + link: '/zh-cn/docs/dev/user_doc/guide/installation/cluster.html', }, { title: 'Kubernetes部署(Kubernetes)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/installation/kubernetes.html', + link: '/zh-cn/docs/dev/user_doc/guide/installation/kubernetes.html', }, { title: '集成', children: [ { title: '基于Rainbond部署(Cluster)', - link: '/zh-cn/docs/3.2.2/user_doc/guide/integration/rainbond.html', + link: '/zh-cn/docs/dev/user_doc/guide/integration/rainbond.html', }, ], }, @@ -1255,27 +1255,27 @@ export default { children: [ { title: '元数据文档', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/metadata.html', + link: '/zh-cn/docs/dev/user_doc/architecture/metadata.html', }, { title: '架构设计', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/design.html', + link: '/zh-cn/docs/dev/user_doc/architecture/design.html', }, { title: '配置文件', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/configuration.html', + link: '/zh-cn/docs/dev/user_doc/architecture/configuration.html', }, { title: '任务结构', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/task-structure.html', + link: '/zh-cn/docs/dev/user_doc/architecture/task-structure.html', }, { title: '负载均衡', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/load-balance.html', + link: '/zh-cn/docs/dev/user_doc/architecture/load-balance.html', }, { title: '缓存', - link: '/zh-cn/docs/3.2.2/user_doc/architecture/cache.html', + link: '/zh-cn/docs/dev/user_doc/architecture/cache.html', }, ], }, @@ -1284,15 +1284,15 @@ export default { children: [ { title: 'API调用', - link: '/zh-cn/docs/3.2.2/user_doc/guide/api/open-api.html', + link: '/zh-cn/docs/dev/user_doc/guide/api/open-api.html', }, { title: 'PyDolphinScheduler', - link: '/zh-cn/docs/3.2.2/user_doc/guide/api/pydolphinscheduler.html', + link: '/zh-cn/docs/dev/user_doc/guide/api/pydolphinscheduler.html', }, { title: '健康检查', - link: '/zh-cn/docs/3.2.2/user_doc/guide/api/healthcheck.html', + link: '/zh-cn/docs/dev/user_doc/guide/api/healthcheck.html', }, ], }, @@ -1304,69 +1304,69 @@ export default { children: [ { title: '报告安全问题', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/security.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/security.html', }, { title: '如何成为 Committer', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/become-a-committer.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/become-a-committer.html', }, { title: '订阅/取消订阅邮件列表', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/subscribe.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/subscribe.html', }, { title: '参与贡献', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/contribute.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/contribute.html', }, { title: '行为准则', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/code-conduct.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/code-conduct.html', }, { title: 'Review Issue or Pull Requests', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/review.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/review.html', }, { title: 'E2E Contribution Guide', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/e2e-guide.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/e2e-guide.html', }, { title: '提交代码', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/submit-code.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/submit-code.html', }, { title: 'License须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/DS-License.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/DS-License.html', }, { title: '文档须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/document.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/document.html', }, { title: 'Issue须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/issue.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/issue.html', }, { title: 'Pull Request须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/pull-request.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/pull-request.html', }, { title: 'Commit Message须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/commit-message.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/commit-message.html', }, { title: '微基准测试须知', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/microbench.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/microbench.html', }, { title: '单元测试编写指南', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/join/unit-test.html', + link: '/zh-cn/docs/dev/user_doc/contribute/join/unit-test.html', }, ], }, { title: '环境搭建', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/3.2.2elopment-environment-setup.html', + link: '/zh-cn/docs/dev/user_doc/contribute/development-environment-setup.html', }, { title: '设计文档', @@ -1377,27 +1377,27 @@ export default { // children: [ { title: '架构设计', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/architecture-design.html', + link: '/zh-cn/docs/dev/user_doc/contribute/architecture-design.html', }, { title: 'Alert SPI', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/backend/spi/alert.html', + link: '/zh-cn/docs/dev/user_doc/contribute/backend/spi/alert.html', }, { title: 'Registry SPI', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/backend/spi/registry.html', + link: '/zh-cn/docs/dev/user_doc/contribute/backend/spi/registry.html', }, { title: 'Task SPI', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/backend/spi/task.html', + link: '/zh-cn/docs/dev/user_doc/contribute/backend/spi/task.html', }, { title: 'Datasource SPI', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/backend/spi/datasource.html', + link: '/zh-cn/docs/dev/user_doc/contribute/backend/spi/datasource.html', }, { title: '组件设计', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/backend/mechanism/overview.html', + link: '/zh-cn/docs/dev/user_doc/contribute/backend/mechanism/overview.html', }, ], }, @@ -1406,33 +1406,33 @@ export default { children: [ { title: '前端开发', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/frontend-3.2.2elopment.html', + link: '/zh-cn/docs/dev/user_doc/contribute/frontend-development.html', }, { title: 'API规范', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/api-standard.html', + link: '/zh-cn/docs/dev/user_doc/contribute/api-standard.html', }, { title: 'E2E 自动化测试', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/e2e-test.html', + link: '/zh-cn/docs/dev/user_doc/contribute/e2e-test.html', }, { title: 'API 自动化测试', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/api-test.html', + link: '/zh-cn/docs/dev/user_doc/contribute/api-test.html', }, { title: '日志规范', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/log-specification.html', + link: '/zh-cn/docs/dev/user_doc/contribute/log-specification.html', }, ], }, { title: '发版', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/release.html', + link: '/zh-cn/docs/dev/user_doc/contribute/release.html', }, { title: '问题与交流', - link: '/zh-cn/docs/3.2.2/user_doc/contribute/have-questions.html', + link: '/zh-cn/docs/dev/user_doc/contribute/have-questions.html', }, ], }, @@ -1441,7 +1441,7 @@ export default { children: [ { title: 'DSIP', - link: '/zh-cn/docs/3.2.2/user_doc/DSIP.html', + link: '/zh-cn/docs/dev/user_doc/DSIP.html', }, ], }, @@ -1450,11 +1450,11 @@ export default { children: [ { title: '通用设置', - link: '/zh-cn/docs/3.2.2/user_doc/guide/howto/general-setting.html', + link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html', }, { title: '数据源设置', - link: '/zh-cn/docs/3.2.2/user_doc/guide/howto/datasource-setting.html', + link: '/zh-cn/docs/dev/user_doc/guide/howto/datasource-setting.html', }, { title: '其他', diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index a490e47d89..cc3bc94fc0 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -191,21 +191,19 @@ The default configuration is as follows: Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`. -For ETCD Registry, please see more details -on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md). -For JDBC Registry, please see more details -on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md). +For ETCD Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md). +For JDBC Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md). ### common.properties [hadoop、s3、yarn config properties] Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location: -| Service | Configuration file | -|---------------|-----------------------------------------------------------------------| -| Master Server | `master-server/conf/common.properties` | -| Api Server | `api-server/conf/common.properties`, `api-server/conf/aws.yaml` | -| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` | -| Alert Server | `alert-server/conf/common.properties` | +| Service | Configuration file | +|---------------|----------------------------------------| +| Master Server | `master-server/conf/common.properties` | +| Api Server | `api-server/conf/common.properties` | +| Worker Server | `worker-server/conf/common.properties` | +| Alert Server | `alert-server/conf/common.properties` | The default configuration is as follows: @@ -214,6 +212,10 @@ The default configuration is as follows: | data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files | | resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE | | resource.upload.path | /dolphinscheduler | storage path of resource files | +| aws.access.key.id | minioadmin | access key id of S3 | +| aws.secret.access.key | minioadmin | secret access key of S3 | +| aws.region | us-east-1 | region of S3 | +| aws.s3.endpoint | http://minio:9000 | endpoint of S3 | | hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS | | fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory | | hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission | diff --git a/docs/docs/en/contribute/release.md b/docs/docs/en/contribute/release.md index 1bd5b48d3a..4fffce4688 100644 --- a/docs/docs/en/contribute/release.md +++ b/docs/docs/en/contribute/release.md @@ -340,7 +340,7 @@ svn --username="${A_USERNAME}" commit -m "release ${VERSION}" #### Check source package -Decompress `apache-dolphinscheduler-3.2.2-src.tar.gz` then check the following items: +Decompress `apache-dolphinscheduler--src.tar.gz` then check the following items: - Check whether source tarball is oversized for including nonessential files - `LICENSE` and `NOTICE` files exist @@ -353,7 +353,7 @@ Decompress `apache-dolphinscheduler-3.2.2-src.tar.gz` then check the following i #### Check binary packages -Decompress `apache-dolphinscheduler-3.2.2-bin.tar.gz` to check the following items: +Decompress `apache-dolphinscheduler--bin.tar.gz` to check the following items: - `LICENSE` and `NOTICE` files exist - Correct year in `NOTICE` file @@ -389,7 +389,7 @@ Announce the vote result: send the result vote e-mail to `dev@dolphinscheduler.a Title： ```txt -[VOTE] Release Apache DolphinScheduler 3.2.2 +[VOTE] Release Apache DolphinScheduler ``` Body： @@ -397,15 +397,15 @@ Body： ```txt Hello DolphinScheduler Community, -This is a call for vote to release Apache DolphinScheduler version 3.2.2 +This is a call for vote to release Apache DolphinScheduler version -Release notes: https://github.com/apache/dolphinscheduler/releases/tag/3.2.2 +Release notes: https://github.com/apache/dolphinscheduler/releases/tag/ -The release candidates: https://dist.apache.org/repos/dist/dev/dolphinscheduler/3.2.2/ +The release candidates: https://dist.apache.org/repos/dist/dev/dolphinscheduler// Maven 2 staging repository: https://repository.apache.org/content/repositories//org/apache/dolphinscheduler/ -Git tag for the release: https://github.com/apache/dolphinscheduler/tree/3.2.2 +Git tag for the release: https://github.com/apache/dolphinscheduler/tree/ Release Commit ID: https://github.com/apache/dolphinscheduler/commit/ @@ -436,13 +436,13 @@ Checklist for reference: Title： ```txt -[RESULT][VOTE] Release Apache DolphinScheduler 3.2.2 +[RESULT][VOTE] Release Apache DolphinScheduler ``` Body： ```txt -The vote to release Apache DolphinScheduler 3.2.2 has passed.Here is the vote result, +The vote to release Apache DolphinScheduler has passed.Here is the vote result, 4 PMC member +1 votes: @@ -478,19 +478,19 @@ and then find DolphinScheduler in [apache staging repositories](https://reposito ### Update Document Website should be present before you send the announce mail this section will tell you how to change the website. For example, -the release version is `3.2.2`, the following updates are required(note it will take effect immediately when the PR is merged): +the release version is ``, the following updates are required(note it will take effect immediately when the PR is merged): - Repository **apache/dolphinscheduler-website**: - - `config/download.json`: add the download of the `3.2.2` release package - - `scripts/conf.sh`: Add new release version `3.2.2` key-value pair to variable `DEV_RELEASE_DOCS_VERSIONS` + - `config/download.json`: add the download of the `` release package + - `scripts/conf.sh`: Add new release version `` key-value pair to variable `DEV_RELEASE_DOCS_VERSIONS` - Repository **apache/dolphinscheduler** (dev branch): - `docs/configs/site.js`: - - `docsLatest`: update to `3.2.2` - - `docs0`: The `text` of two places of `en-us/zh-cn` needs to be updated to `latest(3.2.2)` - - `docs/configs/index.md.jsx`: Add `3.2.2: docsxyzConfig` and add new `import` for the new `docsxyzConfig` - - `docs/docs/en/history-versions.md` and `docs/docs/zh/history-versions.md`: Add new `3.2.2` release docs. + - `docsLatest`: update to `` + - `docs0`: The `text` of two places of `en-us/zh-cn` needs to be updated to `latest()` + - `docs/configs/index.md.jsx`: Add `: docsxyzConfig` and add new `import` for the new `docsxyzConfig` + - `docs/docs/en/history-versions.md` and `docs/docs/zh/history-versions.md`: Add new `` release docs. - `.github/ISSUE_TEMPLATE/bug-report.yml`: DolphinScheduler's GitHub [bug-report](https://github.com/apache/dolphinscheduler/blob/dev/.github/ISSUE_TEMPLATE/bug-report.yml) - issue template have **Version** selection bottom. So after released we should add the new `3.2.2` to + issue template have **Version** selection bottom. So after released we should add the new `` to bug-report.yml ### Publish Docker Image and Helm Chart @@ -511,7 +511,7 @@ Announcement e-mail template as below： Title： ```txt -[ANNOUNCE] Release Apache DolphinScheduler 3.2.2 +[ANNOUNCE] Release Apache DolphinScheduler ``` Body： @@ -519,7 +519,7 @@ Body： ```txt Hi all, -We are glad to announce the release of Apache DolphinScheduler 3.2.2. Once again I would like to express my thanks to your help. +We are glad to announce the release of Apache DolphinScheduler . Once again I would like to express my thanks to your help. Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduler system, dedicated to solving the complex task dependencies in data processing, making the scheduler system out of the box for data processing. @@ -527,14 +527,14 @@ dedicated to solving the complex task dependencies in data processing, making th Download Links: https://dolphinscheduler.apache.org/en-us/download -Release Notes: https://github.com/apache/dolphinscheduler/releases/tag/3.2.2 +Release Notes: https://github.com/apache/dolphinscheduler/releases/tag/ Website: https://dolphinscheduler.apache.org/ DolphinScheduler Resources: - Issue: https://github.com/apache/dolphinscheduler/issues/ - Mailing list: dev@dolphinscheduler.apache.org -- Documents: https://dolphinscheduler.apache.org/en-us/docs/3.2.2/about/introduction +- Documents: https://dolphinscheduler.apache.org/en-us/docs//about/introduction ``` ## News diff --git a/docs/docs/en/guide/expansion-reduction.md b/docs/docs/en/guide/expansion-reduction.md index 45303dd704..5c9dd7d1d2 100644 --- a/docs/docs/en/guide/expansion-reduction.md +++ b/docs/docs/en/guide/expansion-reduction.md @@ -30,9 +30,9 @@ Attention: DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but w mkdir -p /opt cd /opt # decompress -tar -zxvf apache-dolphinscheduler-3.2.2-bin.tar.gz -C /opt +tar -zxvf apache-dolphinscheduler--bin.tar.gz -C /opt cd /opt -mv apache-dolphinscheduler-3.2.2-bin dolphinscheduler +mv apache-dolphinscheduler--bin dolphinscheduler ``` ```markdown diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index dd1ad77238..a6587a5855 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -233,9 +233,9 @@ kubectl scale --replicas=6 sts dolphinscheduler-worker -n test # with test names 2. Create a new `Dockerfile` to add MySQL driver: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-: # For example -# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools:3.2.2 +# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools: # Attention Please, If the build is dolphinscheduler-tools image # You need to change the following line to: COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/tools/libs @@ -284,9 +284,9 @@ externalDatabase: 2. Create a new `Dockerfile` to add MySQL or Oracle driver: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-: # For example -# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: # If you want to support MySQL Datasource COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/libs @@ -316,7 +316,7 @@ docker build -t apache/dolphinscheduler-:new-driver . 1. Create a new `Dockerfile` to install pip: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: COPY requirements.txt /tmp RUN apt-get update && \ apt-get install -y --no-install-recommends python-pip && \ @@ -351,7 +351,7 @@ docker build -t apache/dolphinscheduler-worker:pip . 1. Create a new `Dockerfile` to install Python 3: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: RUN apt-get update && \ apt-get install -y --no-install-recommends python3 && \ rm -rf /var/lib/apt/lists/* diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index e4b0fb86cd..bd2d73f1ce 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -12,7 +12,7 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c ## Preparation - JDK：download [JDK][jdk] (1.8+), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment. -- Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/3.2.2). +- Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/). ## Start DolphinScheduler Standalone Server diff --git a/docs/docs/en/guide/parameter/global.md b/docs/docs/en/guide/parameter/global.md index 46e9fb151d..86cfd69929 100644 --- a/docs/docs/en/guide/parameter/global.md +++ b/docs/docs/en/guide/parameter/global.md @@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case, ### Save the workflow and set global parameters -Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it. +Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it. ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png) diff --git a/docs/docs/en/guide/parameter/project-parameter.md b/docs/docs/en/guide/parameter/project-parameter.md index f9509114d7..235cd23512 100644 --- a/docs/docs/en/guide/parameter/project-parameter.md +++ b/docs/docs/en/guide/parameter/project-parameter.md @@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project. ### Define project-level parameters -On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below: +On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below: ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png) diff --git a/docs/docs/en/guide/parameter/startup-parameter.md b/docs/docs/en/guide/parameter/startup-parameter.md index 11cf5a9f63..bf3acc2034 100644 --- a/docs/docs/en/guide/parameter/startup-parameter.md +++ b/docs/docs/en/guide/parameter/startup-parameter.md @@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config ## Usage -Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters. +Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters. ## Example diff --git a/docs/docs/en/guide/resource/configuration.md b/docs/docs/en/guide/resource/configuration.md index 1c5b389302..6bee9e5a67 100644 --- a/docs/docs/en/guide/resource/configuration.md +++ b/docs/docs/en/guide/resource/configuration.md @@ -28,37 +28,74 @@ The configuration you may need to change: ## connect AWS S3 -if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following: +if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: config the following fields ```properties +...... resource.storage.type=S3 + +...... + +resource.aws.access.key.id=aws_access_key_id +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=aws_secret_access_key +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=us-west-2 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint= + +...... ``` -```yaml -aws: - s3: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - bucket.name: - endpoint: +## Use HDFS or Remote Object Storage -``` - -## connect OSS S3 - -if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: - -config the following fields +After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. ```properties +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# resource view suffixs +#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js + +# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS +resource.storage.type=LOCAL +# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended +resource.storage.upload.base.path=/tmp/dolphinscheduler + +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=minioadmin +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=minioadmin +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=cn-north-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://localhost:9000 + # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -70,24 +107,89 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com -``` - -## connect OBS S3 - -if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: - -config the following fields - -```properties -# access key id, required if you set resource.storage.type=OBS +# alibaba cloud access key id, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.id= -# access key secret, required if you set resource.storage.type=OBS +# alibaba cloud access key secret, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.secret= # oss bucket name, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path +resource.hdfs.root.user=hdfs +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +resource.hdfs.fs.defaultFS=hdfs://mycluster:8020 + +# whether to startup kerberos +hadoop.security.authentication.startup.state=false + +# java.security.krb5.conf path +java.security.krb5.conf.path=/opt/krb5.conf + +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM + +# login user from keytab path +login.user.keytab.path=/opt/hdfs.headless.keytab + +# kerberos expire time, the unit is hour +kerberos.expire.time=2 + + +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) +yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s + +# datasource encryption enable +datasource.encryption.enable=false + +# datasource encryption salt +datasource.encryption.salt=!@#$%^&* + +# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in +# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server +# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`). +data-quality.jar.dir= + +#data-quality.error.output.path=/tmp/data-quality-error-data + +# Network IP gets priority, default inner outer + +# Whether hive SQL is executed in the same session +support.hive.oneSession=false + +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=true + +# network interface preferred like eth0, default: empty +#dolphin.scheduler.network.interface.preferred= + +# network IP gets priority, default: inner outer +#dolphin.scheduler.network.priority.strategy=default + +# system env path +#dolphinscheduler.env.path=dolphinscheduler_env.sh + +# development state +development.state=false + +# rpc port +alert.rpc.port=50052 + +# set path of conda.sh +conda.path=/opt/anaconda3/etc/profile.d/conda.sh + +# Task resource limit state +task.resource.limit.state=false + +# way to collect applicationId: log(original regex match), aop +appId.collect: log ``` > **Note:** diff --git a/docs/docs/en/guide/start/docker.md b/docs/docs/en/guide/start/docker.md index 9470e304e6..f29bcd5c75 100644 --- a/docs/docs/en/guide/start/docker.md +++ b/docs/docs/en/guide/start/docker.md @@ -19,7 +19,7 @@ Start DolphinScheduler with standalone-server Docker images is the easiest way t you can learn DolphinScheduler's concepts and usage, with minimal cost. ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= $ docker run --name dolphinscheduler-standalone-server -p 12345:12345 -p 25333:25333 -d apache/dolphinscheduler-standalone-server:"${DOLPHINSCHEDULER_VERSION}" ``` @@ -37,11 +37,11 @@ be stored on disks after you change docker-compose configuration, and it is robu DolphinScheduler in a long term. You have to install [docker-compose](https://docs.docker.com/compose/install/) before you start servers. -After complete the installation, get the `docker-compose.yaml` file from [download page](https://dolphinscheduler.apache.org/en-us/download/3.2.2) +After complete the installation, get the `docker-compose.yaml` file from [download page](https://dolphinscheduler.apache.org/en-us/download/) form its source package, and make sure you get the right version. After download the package, you can run the commands as below. ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= $ tar -zxf apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src.tar.gz # Going to docker-compose's location # For Mac or Linux users @@ -68,7 +68,7 @@ $ docker-compose --profile all up -d container when it up. You could start DolphinScheduler server separately if you want to reuse your exists services. ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= # Initialize the database, make sure database already exists $ docker run -d --name dolphinscheduler-tools \ -e DATABASE="postgresql" \ @@ -134,5 +134,5 @@ and use `admin` and `dolphinscheduler123` as default username and password in th You can modify some environment variables to change configurations when you are starting servers through Docker. We have an example in [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper) to change database and ZooKeeper configurations, -and you could find all environment variables in [all environment variables](https://github.com/apache/dolphinscheduler/blob/3.2.2/script/env/dolphinscheduler_env.sh) +and you could find all environment variables in [all environment variables](https://github.com/apache/dolphinscheduler/blob//script/env/dolphinscheduler_env.sh) and change them if you want. diff --git a/docs/docs/en/guide/task/dms.md b/docs/docs/en/guide/task/dms.md index 89cf2ef4db..a19ec4ba4b 100644 --- a/docs/docs/en/guide/task/dms.md +++ b/docs/docs/en/guide/task/dms.md @@ -73,17 +73,14 @@ Parameters of restarting the task by interface ## Environment to prepare -Some AWS configuration is required, modify a field in file `aws.yaml` +Some AWS configuration is required, modify a field in file `common.properties` ```yaml -dms: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - endpoint: +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id= +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key= +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region= ``` diff --git a/docs/docs/en/guide/task/sagemaker.md b/docs/docs/en/guide/task/sagemaker.md index b520dd33cc..7782377130 100644 --- a/docs/docs/en/guide/task/sagemaker.md +++ b/docs/docs/en/guide/task/sagemaker.md @@ -35,17 +35,14 @@ The task plugin are shown as follows: ## Environment to prepare -Some AWS configuration is required, modify a field in file `aws.yaml` +Some AWS configuration is required, modify a field in file `common.properties` ```yaml -sagemaker: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - endpoint: +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id= +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key= +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region= ``` diff --git a/docs/docs/en/history-versions.md b/docs/docs/en/history-versions.md index e0eceab968..71035b53a1 100644 --- a/docs/docs/en/history-versions.md +++ b/docs/docs/en/history-versions.md @@ -4,10 +4,6 @@ #### Setup instructions, are available for each stable version of Apache DolphinScheduler below: -### Versions: 3.2.2 - -#### Links： [3.2.2 Document](../3.2.2/user_doc/about/introduction.md) - ### Versions: 3.2.1 #### Links： [3.2.1 Document](../3.2.1/user_doc/about/introduction.md) diff --git a/docs/docs/zh/contribute/release.md b/docs/docs/zh/contribute/release.md index c68bf4ced2..5f4f4bdf7c 100644 --- a/docs/docs/zh/contribute/release.md +++ b/docs/docs/zh/contribute/release.md @@ -347,7 +347,7 @@ svn --username="${A_USERNAME}" commit -m "release ${VERSION}" #### 检查源码包的文件内容 -解压缩`apache-dolphinscheduler-3.2.2-src.tar.gz`，进行如下检查: +解压缩`apache-dolphinscheduler--src.tar.gz`，进行如下检查: - 检查源码包是否包含由于包含不必要文件，致使 tarball 过于庞大 - 存在`LICENSE`和`NOTICE`文件 @@ -359,7 +359,7 @@ svn --username="${A_USERNAME}" commit -m "release ${VERSION}" #### 检查二进制包的文件内容 -解压缩`apache-dolphinscheduler-3.2.2-bin.tar.gz`进行如下检查: +解压缩`apache-dolphinscheduler--bin.tar.gz`进行如下检查: - 存在`LICENSE`和`NOTICE`文件 - 所有文本文件开头都有 ASF 许可证 @@ -394,7 +394,7 @@ DolphinScheduler 社区投票，发起投票邮件到`dev@dolphinscheduler.apach 标题： ```txt -[VOTE] Release Apache DolphinScheduler 3.2.2 +[VOTE] Release Apache DolphinScheduler ``` 正文： @@ -402,15 +402,15 @@ DolphinScheduler 社区投票，发起投票邮件到`dev@dolphinscheduler.apach ```txt Hello DolphinScheduler Community, -This is a call for vote to release Apache DolphinScheduler version 3.2.2 +This is a call for vote to release Apache DolphinScheduler version -Release notes: https://github.com/apache/dolphinscheduler/releases/tag/3.2.2 +Release notes: https://github.com/apache/dolphinscheduler/releases/tag/ -The release candidates: https://dist.apache.org/repos/dist/dev/dolphinscheduler/3.2.2/ +The release candidates: https://dist.apache.org/repos/dist/dev/dolphinscheduler// Maven 2 staging repository: https://repository.apache.org/content/repositories//org/apache/dolphinscheduler/ -Git tag for the release: https://github.com/apache/dolphinscheduler/tree/3.2.2 +Git tag for the release: https://github.com/apache/dolphinscheduler/tree/ Release Commit ID: https://github.com/apache/dolphinscheduler/commit/ @@ -441,11 +441,11 @@ Checklist for reference: Title： ```txt -[RESULT][VOTE] Release Apache DolphinScheduler 3.2.2 +[RESULT][VOTE] Release Apache DolphinScheduler ``` ```txt -The vote to release Apache DolphinScheduler 3.2.2 has passed.Here is the vote result, +The vote to release Apache DolphinScheduler has passed.Here is the vote result, 4 PMC member +1 votes: @@ -480,17 +480,17 @@ git push --delete "${GH_REMOTE}" "${VERSION}-prepare" ### 更新文档 -官网应该在您发送通知邮件之前完成更新，本节将告诉您如何更改网站。假设发版的版本是 `3.2.2`，需要进行以下更新（注意，当修改 pull requests 被 merge 后就会生效）: +官网应该在您发送通知邮件之前完成更新，本节将告诉您如何更改网站。假设发版的版本是 ``，需要进行以下更新（注意，当修改 pull requests 被 merge 后就会生效）: - **apache/dolphinscheduler-website** 仓库： - - `config/download.json`: 增加 `3.2.2` 版本发布包的下载 - - `scripts/conf.sh`: 在变量 `DEV_RELEASE_DOCS_VERSIONS` 中增加版本为 `3.2.2` 的新键值对 + - `config/download.json`: 增加 `` 版本发布包的下载 + - `scripts/conf.sh`: 在变量 `DEV_RELEASE_DOCS_VERSIONS` 中增加版本为 `` 的新键值对 - **apache/dolphinscheduler** 仓库 (dev 分支)： - `docs/configs/site.js`: - - `docsLatest`: 更新为 `3.2.2` - - `docs0`: 两处 `en-us/zh-cn` 的 `text` 更新为 `latest(3.2.2)` - - `docs/configs/index.md.jsx`: 增加 `'3.2.2': docsxyzConfig,` 以及新的 `import` - - `docs/docs/en/history-versions.md` 和 `docs/docs/zh/history-versions.md`: 增加新的发版版本 `3.2.2` 的链接 + - `docsLatest`: 更新为 `` + - `docs0`: 两处 `en-us/zh-cn` 的 `text` 更新为 `latest()` + - `docs/configs/index.md.jsx`: 增加 `'': docsxyzConfig,` 以及新的 `import` + - `docs/docs/en/history-versions.md` 和 `docs/docs/zh/history-versions.md`: 增加新的发版版本 `` 的链接 - `.github/ISSUE_TEMPLATE/bug-report.yml`: DolphinScheduler 在 GitHub bug report 的 issue 中有版本选择，当有新的版本发版后，需要更新 [bug-report](https://github.com/apache/dolphinscheduler/blob/dev/.github/ISSUE_TEMPLATE/bug-report.yml) 中的 **Version** 部分。 @@ -510,7 +510,7 @@ git push --delete "${GH_REMOTE}" "${VERSION}-prepare" 标题： ```txt -[ANNOUNCE] Release Apache DolphinScheduler 3.2.2 +[ANNOUNCE] Release Apache DolphinScheduler ``` 正文： @@ -518,7 +518,7 @@ git push --delete "${GH_REMOTE}" "${VERSION}-prepare" ```txt Hi all, -We are glad to announce the release of Apache DolphinScheduler 3.2.2. Once again I would like to express my thanks to your help. +We are glad to announce the release of Apache DolphinScheduler . Once again I would like to express my thanks to your help. Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduler system, dedicated to solving the complex task dependencies in data processing, making the scheduler system out of the box for data processing. @@ -526,14 +526,14 @@ dedicated to solving the complex task dependencies in data processing, making th Download Links: https://dolphinscheduler.apache.org/zh-cn/download -Release Notes: https://github.com/apache/dolphinscheduler/releases/tag/3.2.2 +Release Notes: https://github.com/apache/dolphinscheduler/releases/tag/ Website: https://dolphinscheduler.apache.org/ DolphinScheduler Resources: - Issue: https://github.com/apache/dolphinscheduler/issues/ - Mailing list: dev@dolphinscheduler.apache.org -- Documents: https://dolphinscheduler.apache.org/zh-cn/docs/3.2.2/about/introduction +- Documents: https://dolphinscheduler.apache.org/zh-cn/docs//about/introduction ``` ## News diff --git a/docs/docs/zh/guide/expansion-reduction.md b/docs/docs/zh/guide/expansion-reduction.md index 36d04bed9f..ad27e32a35 100644 --- a/docs/docs/zh/guide/expansion-reduction.md +++ b/docs/docs/zh/guide/expansion-reduction.md @@ -30,9 +30,9 @@ mkdir -p /opt cd /opt # 解压缩 -tar -zxvf apache-dolphinscheduler-3.2.2-bin.tar.gz -C /opt +tar -zxvf apache-dolphinscheduler--bin.tar.gz -C /opt cd /opt -mv apache-dolphinscheduler-3.2.2-bin dolphinscheduler +mv apache-dolphinscheduler--bin dolphinscheduler ``` ```markdown diff --git a/docs/docs/zh/guide/installation/kubernetes.md b/docs/docs/zh/guide/installation/kubernetes.md index 4846d4b3e3..f4b95de27b 100644 --- a/docs/docs/zh/guide/installation/kubernetes.md +++ b/docs/docs/zh/guide/installation/kubernetes.md @@ -232,9 +232,9 @@ kubectl scale --replicas=6 sts dolphinscheduler-worker -n test # with test names 2. 创建一个新的 `Dockerfile`，用于添加 MySQL 的驱动包: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-: # 例如 -# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools:3.2.2 +# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools: # 注意，如果构建的是dolphinscheduler-tools镜像 # 需要将下面一行修改为COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/tools/libs @@ -283,9 +283,9 @@ externalDatabase: 2. 创建一个新的 `Dockerfile`，用于添加 MySQL 或者 Oracle 驱动包: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-: # 例如 -# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +# FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: # 如果你想支持 MySQL 数据源 COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/libs @@ -315,7 +315,7 @@ docker build -t apache/dolphinscheduler-:new-driver . 1. 创建一个新的 `Dockerfile`，用于安装 pip: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: COPY requirements.txt /tmp RUN apt-get update && \ apt-get install -y --no-install-recommends python-pip && \ @@ -350,7 +350,7 @@ docker build -t apache/dolphinscheduler-worker:pip . 1. 创建一个新的 `Dockerfile`，用于安装 Python 3: ``` -FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.2.2 +FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker: RUN apt-get update && \ apt-get install -y --no-install-recommends python3 && \ rm -rf /var/lib/apt/lists/* diff --git a/docs/docs/zh/guide/installation/standalone.md b/docs/docs/zh/guide/installation/standalone.md index 34cbe8d6b7..dbd9ca94b6 100644 --- a/docs/docs/zh/guide/installation/standalone.md +++ b/docs/docs/zh/guide/installation/standalone.md @@ -10,7 +10,7 @@ Standalone 仅适用于 DolphinScheduler 的快速体验. ## 前置准备工作 - JDK：下载[JDK][jdk] (1.8+)，安装并配置 `JAVA_HOME` 环境变量，并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在，可以跳过这步。 -- 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/3.2.2)下载 DolphinScheduler 二进制包 +- 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/)下载 DolphinScheduler 二进制包 ## 启动 DolphinScheduler Standalone Server diff --git a/docs/docs/zh/guide/parameter/global.md b/docs/docs/zh/guide/parameter/global.md index 3b8b307f91..cd26a291b0 100644 --- a/docs/docs/zh/guide/parameter/global.md +++ b/docs/docs/zh/guide/parameter/global.md @@ -20,7 +20,7 @@ ### 保存工作流，并设置全局参数 -全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示： +全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示： ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png) diff --git a/docs/docs/zh/guide/parameter/project-parameter.md b/docs/docs/zh/guide/parameter/project-parameter.md index 8692b198a8..df26e7a7df 100644 --- a/docs/docs/zh/guide/parameter/project-parameter.md +++ b/docs/docs/zh/guide/parameter/project-parameter.md @@ -8,7 +8,7 @@ ### 定义项目级别参数 -在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示： +在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示： ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png) diff --git a/docs/docs/zh/guide/parameter/startup-parameter.md b/docs/docs/zh/guide/parameter/startup-parameter.md index 38e15787d0..7da65657a7 100644 --- a/docs/docs/zh/guide/parameter/startup-parameter.md +++ b/docs/docs/zh/guide/parameter/startup-parameter.md @@ -6,7 +6,7 @@ ## 使用方式 -启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。 +启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。 ## 任务样例 diff --git a/docs/docs/zh/guide/resource/configuration.md b/docs/docs/zh/guide/resource/configuration.md index 735c75da97..19a34c577c 100644 --- a/docs/docs/zh/guide/resource/configuration.md +++ b/docs/docs/zh/guide/resource/configuration.md @@ -26,35 +26,77 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用 ## 对接AWS S3 -如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下： +如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： 配置以下字段 ```properties +...... resource.storage.type=S3 + +...... + +resource.aws.access.key.id=aws_access_key_id +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=aws_secret_access_key +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=us-west-2 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint= + +...... ``` -```yaml -aws: - s3: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - bucket.name: - endpoint: +## 对接分布式或远端对象存储 -``` +当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置： -## 对接阿里云 OSS +### 配置 common.properties 文件 -如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： +在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： ```properties +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS +resource.storage.type=LOCAL + +# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration, +# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended +resource.storage.upload.base.path=/tmp/dolphinscheduler + +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=minioadmin +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=minioadmin +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=cn-north-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://localhost:9000 + # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -66,22 +108,87 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com -``` - -## 对接华为云 OBS - -如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： - -```properties -# access key id, required if you set resource.storage.type=OBS +# alibaba cloud access key id, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.id= -# access key secret, required if you set resource.storage.type=OBS +# alibaba cloud access key secret, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.secret= # oss bucket name, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path +resource.hdfs.root.user=root +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; +# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +resource.hdfs.fs.defaultFS=hdfs://localhost:8020 + +# whether to startup kerberos +hadoop.security.authentication.startup.state=false + +# java.security.krb5.conf path +java.security.krb5.conf.path=/opt/krb5.conf + +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM + +# login user from keytab path +login.user.keytab.path=/opt/hdfs.headless.keytab + +# kerberos expire time, the unit is hour +kerberos.expire.time=2 +# resource view suffixs +#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js + +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; +# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) +yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s + +# datasource encryption enable +datasource.encryption.enable=false + +# datasource encryption salt +datasource.encryption.salt=!@#$%^&* + +# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in +# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server +# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`). +data-quality.jar.dir= + +#data-quality.error.output.path=/tmp/data-quality-error-data + +# Network IP gets priority, default inner outer + +# Whether hive SQL is executed in the same session +support.hive.oneSession=false + +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; +# if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=true + +# network interface preferred like eth0, default: empty +#dolphin.scheduler.network.interface.preferred= + +# network IP gets priority, default: inner outer +#dolphin.scheduler.network.priority.strategy=default + +# system env path +#dolphinscheduler.env.path=env/dolphinscheduler_env.sh + +# development state +development.state=false + +# rpc port +alert.rpc.port=50052 + +# way to collect applicationId: log(original regex match), aop +appId.collect: log ``` > **注意**： diff --git a/docs/docs/zh/guide/start/docker.md b/docs/docs/zh/guide/start/docker.md index 3ab1d23cb8..2a3a577a73 100644 --- a/docs/docs/zh/guide/start/docker.md +++ b/docs/docs/zh/guide/start/docker.md @@ -18,7 +18,7 @@ 你可以最快速的体验到 DolphinScheduler 的大部分功能，了解主要和概念和内容。 ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= $ docker run --name dolphinscheduler-standalone-server -p 12345:12345 -p 25333:25333 -d apache/dolphinscheduler-standalone-server:"${DOLPHINSCHEDULER_VERSION}" ``` @@ -33,11 +33,11 @@ $ docker run --name dolphinscheduler-standalone-server -p 12345:12345 -p 25333:2 服务重启的时候保留元数据（如需要挂载到本地路径需要做指定）。他更健壮，能保证用户体验更加完整的 DolphinScheduler 服务。这种方式需要先安装 [docker-compose](https://docs.docker.com/compose/install/)，链接适用于 Mac，Linux，Windows。 -确保 docker-compose 顺利安装后，需要获取 `docker-compose.yaml` 文件，通过[下载页面](https://dolphinscheduler.apache.org/en-us/download/3.2.2) +确保 docker-compose 顺利安装后，需要获取 `docker-compose.yaml` 文件，通过[下载页面](https://dolphinscheduler.apache.org/en-us/download/) 下载对应版本源码包可能是最快的方法，当下载完源码后就可以运行命令进行部署了。 ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= $ tar -zxf apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src.tar.gz # Mac Linux 用户 $ cd apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src/deploy/docker @@ -61,7 +61,7 @@ $ docker-compose --profile all up -d ZooKeeper 且不想启动新的服务，可以使用这个方式分别启动 DolphinScheduler 容器。 ```shell -$ DOLPHINSCHEDULER_VERSION=3.2.2 +$ DOLPHINSCHEDULER_VERSION= # 初始化数据库，其确保数据库 已经存在 $ docker run -d --name dolphinscheduler-tools \ -e DATABASE="postgresql" \ diff --git a/docs/docs/zh/guide/task/appendix.md b/docs/docs/zh/guide/task/appendix.md index 9cc9002dd3..6ba27ad38b 100644 --- a/docs/docs/zh/guide/task/appendix.md +++ b/docs/docs/zh/guide/task/appendix.md @@ -8,7 +8,7 @@ |----------|--------------------------------------------------------------------------------------------------------------------------------------| | 任务名称 | 任务的名称，同一个工作流定义中的节点名称不能重复。 | | 运行标志 | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。 | -| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。 | +| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。 | | 描述 | 当前节点的功能描述。 | | 任务优先级 | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。 | | Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。 | diff --git a/docs/docs/zh/guide/task/dms.md b/docs/docs/zh/guide/task/dms.md index 8a87a36f9a..6013f45aef 100644 --- a/docs/docs/zh/guide/task/dms.md +++ b/docs/docs/zh/guide/task/dms.md @@ -73,17 +73,14 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS ## 环境配置 -需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息 +需要进行AWS的一些配置，修改`common.properties`中的以下配置信息 ```yaml -dms: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - endpoint: +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id= +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key= +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region= ``` diff --git a/docs/docs/zh/guide/task/sagemaker.md b/docs/docs/zh/guide/task/sagemaker.md index 22927171e9..e4b4c61542 100644 --- a/docs/docs/zh/guide/task/sagemaker.md +++ b/docs/docs/zh/guide/task/sagemaker.md @@ -33,17 +33,14 @@ DolphinScheduler SageMaker 组件的功能: ## 环境配置 -需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息 +需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息 ```yaml -sagemaker: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: - access.key.secret: - region: - endpoint: +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id= +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key= +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region= ``` diff --git a/docs/docs/zh/history-versions.md b/docs/docs/zh/history-versions.md index 3bbb36a4d4..332a0bb58c 100644 --- a/docs/docs/zh/history-versions.md +++ b/docs/docs/zh/history-versions.md @@ -4,10 +4,6 @@ #### 以下是Apache DolphinScheduler每个稳定版本的设置说明。 -### Versions: 3.2.2 - -#### Links： [3.2.2 Document](../3.2.2/user_doc/about/introduction.md) - ### Versions: 3.2.1 #### Links： [3.2.1 Document](../3.2.1/user_doc/about/introduction.md) diff --git a/docs/img/new_ui/dev/parameter/context-subprocess02.png b/docs/img/new_ui/dev/parameter/context-subprocess02.png index 7d7cd850ea..8c1acf5f11 100644 Binary files a/docs/img/new_ui/dev/parameter/context-subprocess02.png and b/docs/img/new_ui/dev/parameter/context-subprocess02.png differ diff --git a/docs/img/new_ui/dev/parameter/context_parameter04.png b/docs/img/new_ui/dev/parameter/context_parameter04.png index e307e607e7..b1dfd8b3cd 100644 Binary files a/docs/img/new_ui/dev/parameter/context_parameter04.png and b/docs/img/new_ui/dev/parameter/context_parameter04.png differ diff --git a/docs/img/new_ui/dev/parameter/global_parameter02.png b/docs/img/new_ui/dev/parameter/global_parameter02.png index 3d6d535311..1eeaf4b43a 100644 Binary files a/docs/img/new_ui/dev/parameter/global_parameter02.png and b/docs/img/new_ui/dev/parameter/global_parameter02.png differ diff --git a/docs/img/new_ui/dev/parameter/project_parameter01.png b/docs/img/new_ui/dev/parameter/project_parameter01.png index 5f6a483d4c..48ea149b40 100644 Binary files a/docs/img/new_ui/dev/parameter/project_parameter01.png and b/docs/img/new_ui/dev/parameter/project_parameter01.png differ diff --git a/docs/img/new_ui/dev/parameter/startup_parameter02.png b/docs/img/new_ui/dev/parameter/startup_parameter02.png index d1819ad78e..cbeb96b658 100644 Binary files a/docs/img/new_ui/dev/parameter/startup_parameter02.png and b/docs/img/new_ui/dev/parameter/startup_parameter02.png differ diff --git a/docs/img/new_ui/dev/parameter/startup_parameter04.png b/docs/img/new_ui/dev/parameter/startup_parameter04.png index b5799b24a5..e32afb4b31 100644 Binary files a/docs/img/new_ui/dev/parameter/startup_parameter04.png and b/docs/img/new_ui/dev/parameter/startup_parameter04.png differ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 2da74b7e81..844a5983df 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -41,12 +41,6 @@ org.apache.dolphinscheduler dolphinscheduler-dao - - - org.apache.dolphinscheduler - dolphinscheduler-storage-api - - diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 24c8fb2f11..bf28193b3f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -56,13 +56,6 @@ conf - - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources - - **/*.yaml - - conf - diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh index d953e04d2f..c668944139 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh @@ -24,7 +24,6 @@ -XX:+PrintGCDetails -Xloggc:gc.log --XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index bf001bec72..40d556a17e 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -74,7 +74,7 @@ org.apache.dolphinscheduler - dolphinscheduler-task-all-prune + dolphinscheduler-task-all diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index 5453f8fb15..c9fdab4d51 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -57,13 +57,6 @@ conf - - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources - - **/*.yaml - - conf - ${basedir}/../dolphinscheduler-ui/dist ./ui diff --git a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh index d953e04d2f..c668944139 100644 --- a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh @@ -24,7 +24,6 @@ -XX:+PrintGCDetails -Xloggc:gc.log --XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 8c86de7fe6..a2a300b946 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -44,8 +44,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.utils.PropertyUtils; import org.apache.commons.lang3.StringUtils; @@ -165,8 +163,10 @@ public class ExecutorController extends BaseController { if (timeout == null) { timeout = Constants.MAX_TASK_TIMEOUT; } - - List startParamList = PropertyUtils.startParamsTransformPropertyList(startParams); + Map startParamMap = null; + if (startParams != null) { + startParamMap = JSONUtils.toMap(startParams); + } if (complementDependentMode == null) { complementDependentMode = ComplementDependentMode.OFF_MODE; @@ -175,7 +175,7 @@ public class ExecutorController extends BaseController { Map result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber, dryRun, + workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, complementDependentMode, version, allLevelDependent, executionOrder); return returnDataList(result); @@ -262,7 +262,10 @@ public class ExecutorController extends BaseController { timeout = Constants.MAX_TASK_TIMEOUT; } - List startParamList = PropertyUtils.startParamsTransformPropertyList(startParams); + Map startParamMap = null; + if (startParams != null) { + startParamMap = JSONUtils.toMap(startParams); + } if (complementDependentMode == null) { log.debug("Parameter complementDependentMode set to {} due to null.", ComplementDependentMode.OFF_MODE); @@ -280,8 +283,7 @@ public class ExecutorController extends BaseController { result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber, - dryRun, + workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, complementDependentMode, null, allLevelDependent, executionOrder); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java index efec970e6f..706c7e939a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java @@ -61,8 +61,7 @@ public class ProjectParameterController extends BaseController { @Operation(summary = "createProjectParameter", description = "CREATE_PROJECT_PARAMETER_NOTES") @Parameters({ @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)), - @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)), - @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class)) + @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)) }) @PostMapping() @ResponseStatus(HttpStatus.CREATED) @@ -70,10 +69,9 @@ public class ProjectParameterController extends BaseController { public Result createProjectParameter(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("projectParameterName") String projectParameterName, - @RequestParam(value = "projectParameterValue") String projectParameterValue, - @RequestParam(value = "projectParameterDataType", defaultValue = "VARCHAR") String projectParameterDataType) { + @RequestParam(value = "projectParameterValue") String projectParameterValue) { return projectParameterService.createProjectParameter(loginUser, projectCode, projectParameterName, - projectParameterValue, projectParameterDataType); + projectParameterValue); } @Operation(summary = "updateProjectParameter", description = "UPDATE_PROJECT_PARAMETER_NOTES") @@ -81,7 +79,6 @@ public class ProjectParameterController extends BaseController { @Parameter(name = "code", description = "PROJECT_PARAMETER_CODE", schema = @Schema(implementation = long.class, example = "123456")), @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)), @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)), - @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class)) }) @PutMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @@ -90,10 +87,9 @@ public class ProjectParameterController extends BaseController { @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("code") Long code, @RequestParam("projectParameterName") String projectParameterName, - @RequestParam(value = "projectParameterValue") String projectParameterValue, - @RequestParam(value = "projectParameterDataType") String projectParameterDataType) { + @RequestParam(value = "projectParameterValue") String projectParameterValue) { return projectParameterService.updateProjectParameter(loginUser, projectCode, code, projectParameterName, - projectParameterValue, projectParameterDataType); + projectParameterValue); } @Operation(summary = "deleteProjectParametersByCode", description = "DELETE_PROJECT_PARAMETER_NOTES") @@ -137,14 +133,13 @@ public class ProjectParameterController extends BaseController { @Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "searchVal", required = false) String searchVal, - @RequestParam(value = "projectParameterDataType", required = false) String projectParameterDataType, @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize) { checkPageParams(pageNo, pageSize); searchVal = ParameterUtils.handleEscapes(searchVal); return projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, - searchVal, projectParameterDataType); + searchVal); } @Operation(summary = "queryProjectParameterByCode", description = "QUERY_PROJECT_PARAMETER_NOTES") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 55ca697412..3166d3e718 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -30,9 +30,7 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import java.util.List; import java.util.Map; /** @@ -59,7 +57,7 @@ public interface ExecutorService { * @param environmentCode environment code * @param runMode run mode * @param timeout timeout - * @param startParamList the global param values which pass to new process instance + * @param startParams the global param values which pass to new process instance * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode * @param executionOrder the execution order when complementing data * @return execute process instance code @@ -73,7 +71,7 @@ public interface ExecutorService { Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, Integer timeout, - List startParamList, Integer expectedParallelismNumber, + Map startParams, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, Integer version, boolean allLevelDependent, ExecutionOrder executionOrder); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java index af2b047488..ef984dfac4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java @@ -23,17 +23,17 @@ import org.apache.dolphinscheduler.dao.entity.User; public interface ProjectParameterService { Result createProjectParameter(User loginUser, long projectCode, String projectParameterName, - String projectParameterValue, String projectParameterDataType); + String projectParameterValue); Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName, - String projectParameterValue, String projectParameterDataType); + String projectParameterValue); Result deleteProjectParametersByCode(User loginUser, long projectCode, long code); Result batchDeleteProjectParametersByCodes(User loginUser, long projectCode, String codes); Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo, - String searchVal, String projectParameterDataType); + String searchVal); Result queryProjectParameterByCode(User loginUser, long projectCode, long code); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 3e613b11fa..dafef79615 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -90,7 +90,6 @@ import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTrigg import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; @@ -205,7 +204,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ * @param environmentCode environment code * @param runMode run mode * @param timeout timeout - * @param startParamList the global param values which pass to new process instance + * @param startParams the global param values which pass to new process instance * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode * @param testFlag testFlag * @param executionOrder the execution order when complementing data @@ -221,7 +220,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, Integer timeout, - List startParamList, Integer expectedParallelismNumber, + Map startParams, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, Integer version, boolean allLevelDependent, ExecutionOrder executionOrder) { @@ -271,7 +270,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ startNodeList, cronTime, warningType, loginUser.getId(), warningGroupId, runMode, processInstancePriority, workerGroup, tenantCode, - environmentCode, startParamList, expectedParallelismNumber, dryRun, testFlag, + environmentCode, startParams, expectedParallelismNumber, dryRun, testFlag, complementDependentMode, allLevelDependent, executionOrder); if (create > 0) { @@ -733,7 +732,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ WarningType warningType, int executorId, Integer warningGroupId, RunMode runMode, Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, - List startParamList, Integer expectedParallelismNumber, int dryRun, + Map startParams, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, boolean allLevelDependent, ExecutionOrder executionOrder) { @@ -762,8 +761,8 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ if (warningType != null) { command.setWarningType(warningType); } - if (CollectionUtils.isNotEmpty(startParamList)) { - cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParamList)); + if (startParams != null && startParams.size() > 0) { + cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParams)); } command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setExecutorId(executorId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e23101da36..17d1d04712 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -68,7 +68,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override @Transactional public Result createProjectParameter(User loginUser, long projectCode, String projectParameterName, - String projectParameterValue, String projectParameterDataType) { + String projectParameterValue) { Result result = new Result(); // check if user have write perm for project @@ -97,7 +97,6 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj .builder() .paramName(projectParameterName) .paramValue(projectParameterValue) - .paramDataType(projectParameterDataType) .code(CodeGenerateUtils.genCode()) .projectCode(projectCode) .userId(loginUser.getId()) @@ -123,7 +122,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override public Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName, - String projectParameterValue, String projectParameterDataType) { + String projectParameterValue) { Result result = new Result(); // check if user have write perm for project @@ -156,7 +155,6 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj projectParameter.setParamName(projectParameterName); projectParameter.setParamValue(projectParameterValue); - projectParameter.setParamDataType(projectParameterDataType); projectParameter.setUpdateTime(new Date()); projectParameter.setOperator(loginUser.getId()); @@ -238,7 +236,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override public Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo, - String searchVal, String projectParameterDataType) { + String searchVal) { Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); @@ -251,8 +249,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj Page page = new Page<>(pageNo, pageSize); IPage iPage = - projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal, - projectParameterDataType); + projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal); List projectParameterList = iPage.getRecords(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index 1fde93e8b9..9c98880740 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -66,6 +66,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.facebook.presto.jdbc.internal.guava.base.Strings; /** * worker group service impl @@ -247,7 +248,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro * @return boolean */ private String checkWorkerGroupAddrList(WorkerGroup workerGroup) { - if (StringUtils.isEmpty(workerGroup.getAddrList())) { + if (Strings.isNullOrEmpty(workerGroup.getAddrList())) { return null; } Map serverMaps = registryClient.getServerMaps(RegistryNodeType.WORKER); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java index 5b77acc099..2c1a62ccc6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java @@ -38,13 +38,8 @@ import org.apache.dolphinscheduler.common.enums.RunMode; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -80,8 +75,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { final String tenantCode = "root"; final Long environmentCode = 4L; final Integer timeout = 5; - final List startParams = - Collections.singletonList(new Property("start", Direct.IN, DataType.VARCHAR, "params")); + final ImmutableMap startParams = ImmutableMap.of("start", "params"); final Integer expectedParallelismNumber = 6; final int dryRun = 7; final int testFlag = 0; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java index ea8bbf67fd..f1bd20bb23 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -49,9 +48,8 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.createProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.any(), - Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value", - DataType.VARCHAR.name()); + Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -60,9 +58,8 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.updateProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), - Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value", - DataType.LONG.name()); + Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -91,9 +88,8 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.queryProjectParameterListPaging(Mockito.any(), Mockito.anyLong(), - Mockito.anyInt(), Mockito.anyInt(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1", - DataType.VARCHAR.name(), 1, 10); + Mockito.anyInt(), Mockito.anyInt(), Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1", 1, 10); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index ef8ffd2768..ca51690ce6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -36,7 +36,6 @@ import org.apache.dolphinscheduler.dao.entity.ProjectParameter; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import java.util.Collections; @@ -77,8 +76,7 @@ public class ProjectParameterServiceTest { // PERMISSION DENIED when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(false); - Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", - DataType.VARCHAR.name()); + Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -90,29 +88,25 @@ public class ProjectParameterServiceTest { try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", - DataType.VARCHAR.name()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); } // PROJECT_PARAMETER_ALREADY_EXISTS when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", - DataType.VARCHAR.name()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); // INSERT DATA ERROR when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value", - DataType.VARCHAR.name()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value", - DataType.VARCHAR.name()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -123,8 +117,7 @@ public class ProjectParameterServiceTest { // NO PERMISSION when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(false); - Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", - DataType.VARCHAR.name()); + Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -134,33 +127,28 @@ public class ProjectParameterServiceTest { when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", - DataType.VARCHAR.name()); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // PROJECT_PARAMETER_ALREADY_EXISTS when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", - DataType.VARCHAR.name()); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); // PROJECT_UPDATE_ERROR when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value", - DataType.VARCHAR.name()); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value", - DataType.LONG.name()); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); assertEquals(Status.SUCCESS.getCode(), result.getCode()); ProjectParameter projectParameter = (ProjectParameter) result.getData(); assertNotNull(projectParameter.getOperator()); assertNotNull(projectParameter.getUpdateTime()); - assertEquals(DataType.LONG.name(), projectParameter.getParamDataType()); } @Test @@ -235,8 +223,7 @@ public class ProjectParameterServiceTest { .thenReturn(false); Result result = - projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null, - DataType.VARCHAR.name()); + projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -248,10 +235,9 @@ public class ProjectParameterServiceTest { Page page = new Page<>(pageNo, pageSize); page.setRecords(Collections.singletonList(getProjectParameter())); - when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any(), any())) - .thenReturn(page); + when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page); result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, - null, null); + null); assertEquals(Status.SUCCESS.getCode(), result.getCode()); } diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml deleted file mode 100644 index 9bad537cd9..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.dolphinscheduler - dolphinscheduler-authentication - dev-SNAPSHOT - - - dolphinscheduler-aws-authentication - - - - - com.amazonaws - aws-java-sdk-emr - - - - com.amazonaws - aws-java-sdk-s3 - - - - com.amazonaws - aws-java-sdk-sagemaker - - - - com.amazonaws - aws-java-sdk-dms - - - - software.amazon.awssdk - datasync - - - - org.slf4j - slf4j-api - provided - - - - - diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java deleted file mode 100644 index c79c70d8c2..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import static org.apache.dolphinscheduler.authentication.aws.AwsConfigurationKeys.AWS_AUTHENTICATION_TYPE; - -import java.util.Map; - -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; - -@Slf4j -@UtilityClass -public class AWSCredentialsProviderFactor { - - public static AWSCredentialsProvider credentialsProvider(Map awsProperties) { - String awsAuthenticationType = awsProperties.getOrDefault( - AWS_AUTHENTICATION_TYPE, AWSCredentialsProviderType.STATIC_CREDENTIALS_PROVIDER.getName()); - AWSCredentialsProviderType awsCredentialsProviderType = - AWSCredentialsProviderType.of(awsAuthenticationType).orElse(null); - if (awsCredentialsProviderType == null) { - throw new IllegalArgumentException( - "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); - } - switch (awsCredentialsProviderType) { - case STATIC_CREDENTIALS_PROVIDER: - return createAWSStaticCredentialsProvider(awsProperties); - case INSTANCE_PROFILE_CREDENTIALS_PROVIDER: - return createInstanceProfileCredentialsProvider(); - default: - throw new IllegalArgumentException( - "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); - } - - } - - private static AWSCredentialsProvider createAWSStaticCredentialsProvider(Map awsProperties) { - String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); - String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); - final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); - AWSStaticCredentialsProvider awsStaticCredentialsProvider = - new AWSStaticCredentialsProvider(basicAWSCredentials); - log.info("AWSStaticCredentialsProvider created successfully"); - return awsStaticCredentialsProvider; - } - - private static AWSCredentialsProvider createInstanceProfileCredentialsProvider() { - InstanceProfileCredentialsProvider instanceProfileCredentialsProvider = - InstanceProfileCredentialsProvider.getInstance(); - log.info("InstanceProfileCredentialsProvider created successfully"); - return instanceProfileCredentialsProvider; - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java deleted file mode 100644 index 9b932f1554..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import java.util.Optional; - -import lombok.Getter; - -@Getter -public enum AWSCredentialsProviderType { - - STATIC_CREDENTIALS_PROVIDER("AWSStaticCredentialsProvider"), - INSTANCE_PROFILE_CREDENTIALS_PROVIDER("InstanceProfileCredentialsProvider"), - ; - - private final String name; - - AWSCredentialsProviderType(String name) { - this.name = name; - } - - public static Optional of(String name) { - if (name == null) { - return Optional.empty(); - } - for (AWSCredentialsProviderType type : values()) { - if (type.getName().equalsIgnoreCase(name)) { - return Optional.of(type); - } - } - return Optional.empty(); - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java deleted file mode 100644 index 2056268f2b..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import java.util.Map; - -import lombok.experimental.UtilityClass; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService; -import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder; - -@UtilityClass -public class AWSDatabaseMigrationServiceClientFactory { - - public AWSDatabaseMigrationService createAWSDatabaseMigrationServiceClient(Map awsProperties) { - AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); - Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); - String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); - - if (endpoint != null && !endpoint.isEmpty()) { - return AWSDatabaseMigrationServiceClientBuilder - .standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) - .withCredentials(awsCredentialsProvider) - .build(); - } else { - return AWSDatabaseMigrationServiceClientBuilder - .standard() - .withCredentials(awsCredentialsProvider) - .withRegion(regions) - .build(); - } - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java deleted file mode 100644 index ea00473b79..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import java.util.Map; - -import lombok.experimental.UtilityClass; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; -import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; - -@UtilityClass -public class AmazonElasticMapReduceClientFactory { - - public AmazonElasticMapReduce createAmazonElasticMapReduceClient(Map awsProperties) { - AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); - Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); - String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); - - if (endpoint != null && !endpoint.isEmpty()) { - return AmazonElasticMapReduceClientBuilder - .standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) - .withCredentials(awsCredentialsProvider) - .build(); - } else { - return AmazonElasticMapReduceClientBuilder - .standard() - .withCredentials(awsCredentialsProvider) - .withRegion(regions) - .build(); - } - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java deleted file mode 100644 index c45e4de9ea..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import java.util.Map; - -import lombok.experimental.UtilityClass; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; - -@UtilityClass -public class AmazonS3ClientFactory { - - public AmazonS3 createAmazonS3Client(Map awsProperties) { - AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); - Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); - String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); - - if (endpoint != null && !endpoint.isEmpty()) { - return AmazonS3ClientBuilder - .standard() - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) - .withCredentials(awsCredentialsProvider) - .build(); - } else { - return AmazonS3ClientBuilder - .standard() - .withCredentials(awsCredentialsProvider) - .withRegion(regions) - .build(); - } - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java deleted file mode 100644 index 6bff921894..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import java.util.Map; - -import lombok.experimental.UtilityClass; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.sagemaker.AmazonSageMaker; -import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder; - -@UtilityClass -public class AmazonSageMakerClientFactory { - - public AmazonSageMaker createAmazonSageMakerClient(Map awsProperties) { - AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); - Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); - String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); - - if (endpoint != null && !endpoint.isEmpty()) { - return AmazonSageMakerClientBuilder - .standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) - .withCredentials(awsCredentialsProvider) - .build(); - } else { - return AmazonSageMakerClientBuilder - .standard() - .withCredentials(awsCredentialsProvider) - .withRegion(regions) - .build(); - } - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java deleted file mode 100644 index 3d2d8677b3..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -public class AwsConfigurationKeys { - - public static final String AWS_AUTHENTICATION_TYPE = "credentials.provider.type"; - public static final String AWS_REGION = "region"; - public static final String AWS_ENDPOINT = "endpoint"; - - public static final String AWS_ACCESS_KEY_ID = "access.key.id"; - public static final String AWS_SECRET = "access.key.secret"; -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java deleted file mode 100644 index b67c7dd840..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.authentication.aws; - -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.datasync.DataSyncClient; - -import java.util.Map; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class DataSyncClientFactory { - - public DataSyncClient createDataSyncClient(Map awsProperties) { - // todo: upgrade the version of aws sdk - String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); - String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); - final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey); - final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials); - - // create a datasync client - return DataSyncClient.builder() - .region(Region.of(awsProperties.get(AwsConfigurationKeys.AWS_REGION))) - .credentialsProvider(awsCredentialsProvider) - .build(); - } - -} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml deleted file mode 100644 index 6d453bb78a..0000000000 --- a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -aws: - s3: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: accessKey123 - access.key.secret: secretKey123 - region: us-east-1 - bucket.name: dolphinscheduler - endpoint: http://s3:9000 - emr: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - sagemaker: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - dms: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - datasync: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - diff --git a/dolphinscheduler-authentication/pom.xml b/dolphinscheduler-authentication/pom.xml deleted file mode 100644 index b49c4d37d0..0000000000 --- a/dolphinscheduler-authentication/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - 4.0.0 - - org.apache.dolphinscheduler - dolphinscheduler - dev-SNAPSHOT - - - dolphinscheduler-authentication - pom - - - dolphinscheduler-aws-authentication - - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - - diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 6c83c580ae..eda9a72e30 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -41,10 +41,6 @@ - - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication - commons-io commons-io diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 0f12f11d00..cc07accc9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -36,7 +36,6 @@ public final class Constants { public static final String COMMON_PROPERTIES_PATH = "/common.properties"; public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; - public static final String AWS_YAML_PATH = "/aws.yaml"; public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; @@ -131,7 +130,8 @@ public final class Constants { */ public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; - public static final String AWS_S3_BUCKET_NAME = "aws.s3.bucket.name"; + public static final String AWS_S3_BUCKET_NAME = "resource.aws.s3.bucket.name"; + public static final String AWS_END_POINT = "resource.aws.s3.endpoint"; public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name"; public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint"; @@ -704,8 +704,19 @@ public final class Constants { public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint"; + /** + * remote logging for S3 + */ + public static final String REMOTE_LOGGING_S3_ACCESS_KEY_ID = "remote.logging.s3.access.key.id"; + + public static final String REMOTE_LOGGING_S3_ACCESS_KEY_SECRET = "remote.logging.s3.access.key.secret"; + public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name"; + public static final String REMOTE_LOGGING_S3_ENDPOINT = "remote.logging.s3.endpoint"; + + public static final String REMOTE_LOGGING_S3_REGION = "remote.logging.s3.region"; + /** * remote logging for GCS */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java index 4fef7b032a..54dba2d5bd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.log.remote; -import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -27,25 +26,41 @@ import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.Map; import lombok.extern.slf4j.Slf4j; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; @Slf4j public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { - private final String bucketName; + private String accessKeyId; - private final AmazonS3 s3Client; + private String accessKeySecret; + + private String region; + + private String bucketName; + + private String endPoint; + + private AmazonS3 s3Client; private static S3RemoteLogHandler instance; private S3RemoteLogHandler() { + accessKeyId = readAccessKeyID(); + accessKeySecret = readAccessKeySecret(); + region = readRegion(); bucketName = readBucketName(); + endPoint = readEndPoint(); s3Client = buildS3Client(); checkBucketNameExists(bucketName); } @@ -59,8 +74,23 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { } protected AmazonS3 buildS3Client() { - Map awsProperties = PropertyUtils.getByPrefix("aws.s3.", ""); - return AmazonS3ClientFactory.createAmazonS3Client(awsProperties); + if (StringUtils.isNotEmpty(endPoint)) { + return AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( + endPoint, Regions.fromName(region).getName())) + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) + .build(); + } else { + return AmazonS3ClientBuilder + .standard() + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) + .withRegion(Regions.fromName(region)) + .build(); + } } @Override @@ -101,8 +131,24 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { } } + protected String readAccessKeyID() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_ID); + } + + protected String readAccessKeySecret() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_SECRET); + } + + protected String readRegion() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_REGION); + } + protected String readBucketName() { - return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME); + return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_BUCKET_NAME); + } + + protected String readEndPoint() { + return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ENDPOINT); } public void checkBucketNameExists(String bucketName) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index aee1589335..82d4de9599 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.utils; -import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; @@ -43,7 +42,7 @@ public class PropertyUtils { private final ImmutablePriorityPropertyDelegate propertyDelegate = new ImmutablePriorityPropertyDelegate( new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), - new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH)); + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); @@ -107,19 +106,6 @@ public class PropertyUtils { return matchedProperties; } - /** - * Get all properties with specified prefix, like: fs., will replace the prefix with newPrefix - */ - public static Map getByPrefix(String prefix, String newPrefix) { - Map matchedProperties = new HashMap<>(); - for (String propName : propertyDelegate.getPropertyKeys()) { - if (propName.startsWith(prefix)) { - matchedProperties.put(propName.replace(prefix, newPrefix), propertyDelegate.get(propName)); - } - } - return matchedProperties; - } - public static Set getSet(String key, Function> transformFunction, Set defaultValue) { return propertyDelegate.get(key, transformFunction, defaultValue); } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index cf1723700e..fdb553b4bc 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -39,6 +39,17 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=minioadmin +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=minioadmin +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=cn-north-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://localhost:9000 + # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -177,7 +188,16 @@ remote.logging.oss.access.key.secret= remote.logging.oss.bucket.name= # oss endpoint, required if you set remote.logging.target=OSS remote.logging.oss.endpoint= - +# s3 access key id, required if you set remote.logging.target=S3 +remote.logging.s3.access.key.id= +# s3 access key secret, required if you set remote.logging.target=S3 +remote.logging.s3.access.key.secret= +# s3 bucket name, required if you set remote.logging.target=S3 +remote.logging.s3.bucket.name= +# s3 endpoint, required if you set remote.logging.target=S3 +remote.logging.s3.endpoint= +# s3 region, required if you set remote.logging.target=S3 +remote.logging.s3.region= # the location of the google cloud credential, required if you set remote.logging.target=GCS remote.logging.google.cloud.storage.credential=/path/to/credential # gcs bucket name, required if you set remote.logging.target=GCS diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml index f413958e64..2cb48750a4 100644 --- a/dolphinscheduler-common/src/main/resources/remote-logging.yaml +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -34,16 +34,28 @@ remote-logging: bucket.name: # oss endpoint, required if you set remote-logging.target=OSS endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: google.cloud.storage: # the location of the google cloud credential, required if you set remote-logging.target=GCS credential: /path/to/credential - # gcs bucket name, required if you set remote-logging.target=GCS + # gcs bucket name, required if you set remote-logging.target=GCS bucket.name: abs: # abs account name, required if you set resource.storage.type=ABS account.name: - # abs account key, required if you set resource.storage.type=ABS + # abs account key, required if you set resource.storage.type=ABS account.key: - # abs container name, required if you set resource.storage.type=ABS + # abs container name, required if you set resource.storage.type=ABS container.name: diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index 8abfb38269..c915197844 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -17,15 +17,12 @@ package org.apache.dolphinscheduler.common.utils; -import static com.google.common.truth.Truth.assertThat; - import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Collections; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -51,14 +48,4 @@ public class PropertyUtilsTest { }, Sets.newHashSet("docker0")); Assertions.assertEquals(Sets.newHashSet("docker0"), networkInterface); } - - @Test - void getByPrefix() { - Map awsProperties = PropertyUtils.getByPrefix("resource.aws.", ""); - assertThat(awsProperties).containsEntry("access.key.id", "minioadmin"); - assertThat(awsProperties).containsEntry("secret.access.key", "minioadmin"); - assertThat(awsProperties).containsEntry("region", "cn-north-1"); - assertThat(awsProperties).containsEntry("s3.bucket.name", "dolphinscheduler"); - assertThat(awsProperties).containsEntry("endpoint", "http://localhost:9000"); - } } diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties index ce8ef3bf4f..7f66a32a23 100644 --- a/dolphinscheduler-common/src/test/resources/common.properties +++ b/dolphinscheduler-common/src/test/resources/common.properties @@ -45,10 +45,6 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 -# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider -# AWSStaticCredentialsProvider: use the access key and secret key to authenticate -# InstanceProfileCredentialsProvider: use the IAM role to authenticate -aws.credentials.provider.type=AWSStaticCredentialsProvider # The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required resource.aws.access.key.id=minioadmin # The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required @@ -58,7 +54,7 @@ resource.aws.region=cn-north-1 # The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. resource.aws.s3.bucket.name=dolphinscheduler # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.endpoint=http://localhost:9000 +resource.aws.s3.endpoint=http://localhost:9000 # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java index 4343d07c29..03e9140145 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java @@ -55,9 +55,6 @@ public class ProjectParameter { @TableField("param_value") private String paramValue; - @TableField("param_data_type") - private String paramDataType; - private Date createTime; private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java index 3f700a9a84..0e7810158c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java @@ -38,8 +38,7 @@ public interface ProjectParameterMapper extends BaseMapper { IPage queryProjectParameterListPaging(IPage page, @Param("projectCode") long projectCode, @Param("projectParameterIds") List projectParameterIds, - @Param("searchName") String searchName, - @Param("projectParameterDataType") String projectParameterDataType); + @Param("searchName") String searchName); List queryByProjectCode(@Param("projectCode") long projectCode); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml index 4e1e972a04..5b22d40a81 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml @@ -19,7 +19,7 @@ - id, param_name, param_value, param_data_type, code, project_code, user_id, operator, create_time, update_time + id, param_name, param_value, code, project_code, user_id, operator, create_time, update_time select - pp.id, param_name, param_value, param_data_type, code, project_code, user_id, operator, pp.create_time, pp.update_time, + pp.id, param_name, param_value, code, project_code, user_id, operator, pp.create_time, pp.update_time, u.user_name as create_user, u2.user_name as modify_user from t_ds_project_parameter pp @@ -69,9 +69,6 @@ OR param_value LIKE concat('%', #{searchName}, '%') ) - - AND param_data_type = #{projectParameterDataType} - order by pp.update_time desc diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index f3e4324049..42c893bb05 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -667,7 +667,6 @@ CREATE TABLE t_ds_project_parameter id int(11) NOT NULL AUTO_INCREMENT, param_name varchar(255) NOT NULL, param_value text NOT NULL, - param_data_type varchar(50) DEFAULT 'VARCHAR', code bigint(20) NOT NULL, project_code bigint(20) NOT NULL, user_id int(11) DEFAULT NULL, @@ -1071,7 +1070,7 @@ CREATE TABLE t_ds_version -- Records of t_ds_version -- ---------------------------- INSERT INTO t_ds_version -VALUES ('1', '3.2.2'); +VALUES ('1', '3.3.0'); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index ef07c90701..b30bc13887 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -668,7 +668,6 @@ CREATE TABLE `t_ds_project_parameter` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', `param_name` varchar(255) NOT NULL COMMENT 'project parameter name', `param_value` text NOT NULL COMMENT 'project parameter value', - `param_data_type` varchar(50) DEFAULT 'VARCHAR' COMMENT 'project parameter data type', `code` bigint(20) NOT NULL COMMENT 'encoding', `project_code` bigint(20) NOT NULL COMMENT 'project code', `user_id` int(11) DEFAULT NULL COMMENT 'creator id', @@ -1045,7 +1044,7 @@ CREATE TABLE `t_ds_version` ( -- ---------------------------- -- Records of t_ds_version -- ---------------------------- -INSERT IGNORE INTO `t_ds_version` VALUES ('1', '3.2.2'); +INSERT IGNORE INTO `t_ds_version` VALUES ('1', '3.3.0'); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index cdfc459bb5..2d224092c4 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -595,7 +595,6 @@ CREATE TABLE t_ds_project_parameter ( id int NOT NULL , param_name varchar(255) NOT NULL , param_value text NOT NULL , - param_data_type varchar(50) DEFAULT 'VARCHAR', code bigint NOT NULL, project_code bigint NOT NULL, user_id int DEFAULT NULL , @@ -1064,7 +1063,7 @@ INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time) VALUES ('default', 'default', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); -- Records of t_ds_queue,default queue name : default -INSERT INTO t_ds_version(version) VALUES ('3.2.2'); +INSERT INTO t_ds_version(version) VALUES ('3.3.0'); -- -- Table structure for table t_ds_plugin_define diff --git a/dolphinscheduler-dao/src/main/resources/sql/soft_version b/dolphinscheduler-dao/src/main/resources/sql/soft_version index be94e6f53d..15a2799817 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/soft_version +++ b/dolphinscheduler-dao/src/main/resources/sql/soft_version @@ -1 +1 @@ -3.2.2 +3.3.0 diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index 000173b397..d10ac6b710 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -53,6 +53,4 @@ END; d// delimiter ; CALL modify_data_t_ds_audit_log_input_entry; -DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; - -ALTER TABLE t_ds_project_parameter ADD `param_data_type` varchar (50) DEFAULT 'VARCHAR' COMMENT 'project parameter data type'; \ No newline at end of file +DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 4b4cb66651..ee06588e58 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -56,6 +56,4 @@ $$ LANGUAGE plpgsql; d// select modify_data_t_ds_audit_log_input_entry(); -DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); - -ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS param_data_type varchar(50) DEFAULT 'VARCHAR'; \ No newline at end of file +DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java index 1c13d95a0e..48e51d9ca3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java @@ -89,7 +89,7 @@ public class ProjectParameterMapperTest extends BaseDaoTest { insertOne(2, "name2", 2); Page page = new Page(1, 3); - IPage res = projectParameterMapper.queryProjectParameterListPaging(page, 1, null, null, null); + IPage res = projectParameterMapper.queryProjectParameterListPaging(page, 1, null, null); Assertions.assertEquals(1, res.getRecords().size()); } } diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index e1152da947..6dd55fca46 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -67,23 +67,6 @@ ${basedir}/../dolphinscheduler-tools/target/tools tools - - libs/ - bin/ - dist-bin/ - - - - ${basedir}/../dolphinscheduler-tools/target/tools/libs - tools/libs - - dolphinscheduler-*.jar - spring-*.jar - - - - ${basedir}/../dolphinscheduler-tools/target/tools/dist-bin - tools/bin @@ -91,6 +74,11 @@ . + + ${basedir}/../dolphinscheduler-ui/dist + ./ui + + ${basedir}/../script bin diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml deleted file mode 100644 index 6d453bb78a..0000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -aws: - s3: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: accessKey123 - access.key.secret: secretKey123 - region: us-east-1 - bucket.name: dolphinscheduler - endpoint: http://s3:9000 - emr: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - sagemaker: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - dms: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - datasync: - # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider - # AWSStaticCredentialsProvider: use the access key and secret key to authenticate - # InstanceProfileCredentialsProvider: use the IAM role to authenticate - credentials.provider.type: AWSStaticCredentialsProvider - access.key.id: minioadmin - access.key.secret: minioadmin - region: cn-north-1 - endpoint: http://localhost:9000 - diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties index 604befdbf8..7583b3293a 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties @@ -37,6 +37,17 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=accessKey123 +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=secretKey123 +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=us-east-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://s3:9000 + # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml index ccfe940a6d..9a46ed02ad 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -34,7 +34,6 @@ services: retries: 120 volumes: - ./common.properties:/opt/dolphinscheduler/conf/common.properties - - ./aws.yaml:/opt/dolphinscheduler/conf/aws.yaml depends_on: s3: condition: service_healthy diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index 8ded68669d..3999f5c9f5 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -31,12 +31,10 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; import java.net.InetSocketAddress; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.Bootstrap; @@ -56,8 +54,7 @@ public class NettyRemotingClient implements AutoCloseable { private final Bootstrap bootstrap = new Bootstrap(); - private final ReentrantLock channelsLock = new ReentrantLock(); - private final Map channels = new ConcurrentHashMap<>(); + private final ConcurrentHashMap channels = new ConcurrentHashMap<>(128); private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -107,10 +104,9 @@ public class NettyRemotingClient implements AutoCloseable { isStarted.compareAndSet(false, true); } - public IRpcResponse sendSync(final Host host, - final Transporter transporter, + public IRpcResponse sendSync(final Host host, final Transporter transporter, final long timeoutMillis) throws InterruptedException, RemotingException { - final Channel channel = getOrCreateChannel(host); + final Channel channel = getChannel(host); if (channel == null) { throw new RemotingException(String.format("connect to : %s fail", host)); } @@ -141,40 +137,36 @@ public class NettyRemotingClient implements AutoCloseable { return iRpcResponse; } - Channel getOrCreateChannel(Host host) { + private Channel getChannel(Host host) { Channel channel = channels.get(host); if (channel != null && channel.isActive()) { return channel; } - try { - channelsLock.lock(); - channel = channels.get(host); - if (channel != null && channel.isActive()) { - return channel; - } - channel = createChannel(host); - channels.put(host, channel); - } finally { - channelsLock.unlock(); - } - return channel; + return createChannel(host, true); } /** * create channel * - * @param host host + * @param host host + * @param isSync sync flag * @return channel */ - Channel createChannel(Host host) { + private Channel createChannel(Host host, boolean isSync) { try { - ChannelFuture future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); - future = future.sync(); - if (future.isSuccess()) { - return future.channel(); - } else { - throw new IllegalArgumentException("connect to host: " + host + " failed", future.cause()); + ChannelFuture future; + synchronized (bootstrap) { + future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); } + if (isSync) { + future.sync(); + } + if (future.isSuccess()) { + Channel channel = future.channel(); + channels.put(host, channel); + return channel; + } + throw new IllegalArgumentException("connect to host: " + host + " failed"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Connect to host: " + host + " failed", e); @@ -197,23 +189,16 @@ public class NettyRemotingClient implements AutoCloseable { } private void closeChannels() { - try { - channelsLock.lock(); - channels.values().forEach(Channel::close); - } finally { - channelsLock.unlock(); + for (Channel channel : this.channels.values()) { + channel.close(); } + this.channels.clear(); } public void closeChannel(Host host) { - try { - channelsLock.lock(); - Channel channel = this.channels.remove(host); - if (channel != null) { - channel.close(); - } - } finally { - channelsLock.unlock(); + Channel channel = this.channels.remove(host); + if (channel != null) { + channel.close(); } } } diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 2580d15ad1..1b99dd97f3 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -46,10 +46,6 @@ org.apache.dolphinscheduler dolphinscheduler-common - - org.apache.dolphinscheduler - dolphinscheduler-datasource-api - org.apache.dolphinscheduler dolphinscheduler-meter @@ -70,7 +66,12 @@ org.apache.dolphinscheduler - dolphinscheduler-task-all-prune + dolphinscheduler-task-all + + + + org.apache.dolphinscheduler + dolphinscheduler-storage-all diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml index f069b07079..d521e53bc2 100644 --- a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml +++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml @@ -56,13 +56,6 @@ conf - - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources - - **/*.yaml - - conf - diff --git a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh index 1171cd7804..c183f31bb1 100644 --- a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh @@ -24,7 +24,6 @@ -XX:+PrintGCDetails -Xloggc:gc.log --XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java similarity index 64% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java index 5bd1a0a0ba..5e757c7858 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java @@ -15,17 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base.utils; +package org.apache.dolphinscheduler.server.master.event; -import org.junit.jupiter.api.Test; +/** + * This exception represent the exception can be recovered, when we get this exception, + * we will move the event to the fail of the queue. + */ +public class StateEventHandleFailure extends Exception { -import com.google.common.truth.Truth; - -class HostTest { - - @Test - void testEquals() { - Truth.assertThat(Host.of("localhost:8080")).isEqualTo(Host.of("localhost:8080")); + public StateEventHandleFailure(String message) { + super(message); } + public StateEventHandleFailure(String message, Throwable throwable) { + super(message, throwable); + } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java index 3e6f3d4ada..08ee6b66c5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java @@ -28,9 +28,10 @@ public interface StateEventHandler { * @param stateEvent given state event. * @throws StateEventHandleException this exception means it can be recovered. * @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop. + * @throws StateEventHandleException this means it can be recovered. */ boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, - StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError; + StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError, StateEventHandleFailure; StateEventType getEventType(); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java index ed56c34501..9b901d7b60 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java @@ -24,7 +24,7 @@ import java.util.function.Supplier; import lombok.experimental.UtilityClass; -import com.google.common.collect.ImmutableSet; +import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index 7c27e8c985..72c52922d7 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -65,6 +65,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.StateEventHandleError; import org.apache.dolphinscheduler.server.master.event.StateEventHandleException; +import org.apache.dolphinscheduler.server.master.event.StateEventHandleFailure; import org.apache.dolphinscheduler.server.master.event.StateEventHandler; import org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; @@ -297,6 +298,13 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { stateEvent, stateEventHandleException); ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); + } catch (StateEventHandleFailure stateEventHandleFailure) { + log.error("State event handle failed, will move event to the tail: {}", + stateEvent, + stateEventHandleFailure); + this.stateEvents.remove(stateEvent); + this.stateEvents.offer(stateEvent); + ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); } catch (Exception e) { // we catch the exception here, since if the state event handle failed, the state event will still // keep @@ -1358,7 +1366,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } } log.info("The dependTasks of task all success, currentTaskCode: {}, dependTaskCodes: {}", - taskCode, Arrays.toString(indirectDepCodeList.toArray())); + taskCode, Arrays.toString(completeTaskSet.toArray())); return DependResult.SUCCESS; } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 76313df02a..67f505d79b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.registry.api.Registry; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; -import org.apache.commons.lang3.time.DurationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; @@ -77,8 +76,8 @@ final class ZookeeperRegistry implements Registry { .connectString(properties.getConnectString()) .retryPolicy(retryPolicy) .namespace(properties.getNamespace()) - .sessionTimeoutMs(DurationUtils.toMillisInt(properties.getSessionTimeout())) - .connectionTimeoutMs(DurationUtils.toMillisInt(properties.getConnectionTimeout())); + .sessionTimeoutMs((int) properties.getSessionTimeout().toMillis()) + .connectionTimeoutMs((int) properties.getConnectionTimeout().toMillis()); final String digest = properties.getDigest(); if (!Strings.isNullOrEmpty(digest)) { @@ -103,10 +102,9 @@ final class ZookeeperRegistry implements Registry { public void start() { client.start(); try { - if (!client.blockUntilConnected(DurationUtils.toMillisInt(properties.getBlockUntilConnected()), - MILLISECONDS)) { + if (!client.blockUntilConnected((int) properties.getBlockUntilConnected().toMillis(), MILLISECONDS)) { client.close(); - throw new RegistryException("zookeeper connect failed in : " + properties.getConnectString() + "ms"); + throw new RegistryException("zookeeper connect timeout: " + properties.getConnectString()); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -122,7 +120,7 @@ final class ZookeeperRegistry implements Registry { @Override public void connectUntilTimeout(@NonNull Duration timeout) throws RegistryException { try { - if (!client.blockUntilConnected(DurationUtils.toMillisInt(timeout), MILLISECONDS)) { + if (!client.blockUntilConnected((int) timeout.toMillis(), MILLISECONDS)) { throw new RegistryException( String.format("Cannot connect to registry in %s s", timeout.getSeconds())); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml index 92902a608c..7c272e55d5 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml @@ -26,5 +26,5 @@ registry: max-retries: 5 session-timeout: 30s connection-timeout: 9s - block-until-connected: 3s + block-until-connected: 600ms digest: ~ diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java index 5f042b7593..afcfae3fd6 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java @@ -45,9 +45,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.PropertyUtils; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.Date; @@ -56,7 +54,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -153,11 +150,9 @@ public class CuringParamsServiceImpl implements CuringParamsService { return new HashMap<>(); } String startParamJson = cmdParam.get(CommandKeyConstants.CMD_PARAM_START_PARAMS); - List propertyList = PropertyUtils.startParamsTransformPropertyList(startParamJson); - if (CollectionUtils.isEmpty(propertyList)) { - return new HashMap<>(); - } - return propertyList.stream().collect(Collectors.toMap(Property::getProp, Function.identity())); + Map startParamMap = JSONUtils.toMap(startParamJson); + return startParamMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + entry -> new Property(entry.getKey(), Direct.IN, DataType.VARCHAR, entry.getValue()))); } @Override @@ -186,7 +181,8 @@ public class CuringParamsServiceImpl implements CuringParamsService { Map prepareParamsMap = new HashMap<>(); // assign value to definedParams here - Map globalParams = setGlobalParamsMap(processInstance); + Map globalParamsMap = setGlobalParamsMap(processInstance); + Map globalParams = ParameterUtils.getUserDefParamsMap(globalParamsMap); // combining local and global parameters Map localParams = parameters.getInputLocalParametersMap(); @@ -291,16 +287,15 @@ public class CuringParamsServiceImpl implements CuringParamsService { Long.toString(taskInstance.getProcessInstance().getProcessDefinition().getProjectCode())); return params; } - private Map setGlobalParamsMap(ProcessInstance processInstance) { - Map globalParamsMap = new HashMap<>(16); + private Map setGlobalParamsMap(ProcessInstance processInstance) { + Map globalParamsMap = new HashMap<>(16); // global params string String globalParamsStr = processInstance.getGlobalParams(); if (globalParamsStr != null) { List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); globalParamsMap - .putAll(globalParamsList.stream() - .collect(Collectors.toMap(Property::getProp, Function.identity()))); + .putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; } @@ -328,7 +323,7 @@ public class CuringParamsServiceImpl implements CuringParamsService { projectParameterList.forEach(projectParameter -> { Property property = new Property(projectParameter.getParamName(), Direct.IN, - Enum.valueOf(DataType.class, projectParameter.getParamDataType()), + DataType.VARCHAR, projectParameter.getParamValue()); result.put(projectParameter.getParamName(), property); }); diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml index 480301718e..db312b7298 100644 --- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml +++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml @@ -86,13 +86,6 @@ conf - - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources - - **/*.yaml - - conf - ${basedir}/../dolphinscheduler-api/src/main/resources diff --git a/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh index 95b283f91e..42f8b1c656 100644 --- a/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh @@ -24,7 +24,6 @@ -XX:+PrintGCDetails -Xloggc:gc.log --XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml index fb422e2a48..90fa9cb5e0 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml @@ -38,10 +38,5 @@ dolphinscheduler-task-api ${project.version} - - - com.aliyun.oss - aliyun-sdk-oss - diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index 3e4b207b50..a13611316e 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -17,18 +17,19 @@ package org.apache.dolphinscheduler.plugin.storage.s3; +import static org.apache.dolphinscheduler.common.constants.Constants.AWS_END_POINT; import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; -import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; @@ -56,7 +57,11 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ListObjectsV2Request; @@ -74,27 +79,71 @@ import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @Data public class S3StorageOperator implements Closeable, StorageOperate { + private String accessKeyId; + + private String accessKeySecret; + + private String region; + private String bucketName; + private String endPoint; + private AmazonS3 s3Client; public S3StorageOperator() { } public void init() { + accessKeyId = readAccessKeyID(); + accessKeySecret = readAccessKeySecret(); + region = readRegion(); bucketName = readBucketName(); + endPoint = readEndPoint(); s3Client = buildS3Client(); checkBucketNameExists(bucketName); } protected AmazonS3 buildS3Client() { - return AmazonS3ClientFactory.createAmazonS3Client(PropertyUtils.getByPrefix("aws.s3.", "")); + if (!StringUtils.isEmpty(endPoint)) { + return AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( + endPoint, region)) + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) + .build(); + } else { + return AmazonS3ClientBuilder + .standard() + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) + .withRegion(region) + .build(); + } + } + + protected String readAccessKeyID() { + return PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID); + } + + protected String readAccessKeySecret() { + return PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY); + } + + protected String readRegion() { + return PropertyUtils.getString(TaskConstants.AWS_REGION); } protected String readBucketName() { return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME); } + protected String readEndPoint() { + return PropertyUtils.getString(AWS_END_POINT); + } + @Override public void close() throws IOException { s3Client.shutdown(); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java index c5930d713d..0c3f75d861 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java @@ -78,7 +78,13 @@ public class S3StorageOperatorTest { public void setUp() throws Exception { s3StorageOperator = Mockito.spy(new S3StorageOperator()); + doReturn(ACCESS_KEY_ID_MOCK).when(s3StorageOperator) + .readAccessKeyID(); + doReturn(ACCESS_KEY_SECRET_MOCK).when(s3StorageOperator) + .readAccessKeySecret(); + doReturn(REGION_MOCK).when(s3StorageOperator).readRegion(); doReturn(BUCKET_NAME_MOCK).when(s3StorageOperator).readBucketName(); + doReturn(END_POINT_MOCK).when(s3StorageOperator).readEndPoint(); Mockito.doReturn(s3Client) .when(s3StorageOperator).buildS3Client(); Mockito.doNothing() @@ -90,6 +96,9 @@ public class S3StorageOperatorTest { @Test public void testInit() { verify(s3StorageOperator, times(1)).buildS3Client(); + Assertions.assertEquals(ACCESS_KEY_ID_MOCK, s3StorageOperator.getAccessKeyId()); + Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, s3StorageOperator.getAccessKeySecret()); + Assertions.assertEquals(REGION_MOCK, s3StorageOperator.getRegion()); Assertions.assertEquals(BUCKET_NAME_MOCK, s3StorageOperator.getBucketName()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml deleted file mode 100644 index eb251c3fb9..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all-prune/pom.xml +++ /dev/null @@ -1,425 +0,0 @@ - - - - 4.0.0 - - org.apache.dolphinscheduler - dolphinscheduler-task-plugin - dev-SNAPSHOT - - - dolphinscheduler-task-all-prune - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-dataquality - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-datax - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-flink - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-flink-stream - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-http - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-mr - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-pigeon - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-procedure - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-python - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-seatunnel - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-shell - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-spark - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-sql - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-sqoop - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-emr - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-zeppelin - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-jupyter - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-k8s - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-mlflow - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-openmldb - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-dvc - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-dinky - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-java - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-sagemaker - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-pytorch - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-hivecli - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-chunjun - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-dms - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-datasync - ${project.version} - - - * - * - - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-kubeflow - ${project.version} - - - * - * - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-linkis - ${project.version} - - - * - * - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-datafactory - ${project.version} - - - * - * - - - - - org.apache.dolphinscheduler - dolphinscheduler-task-remoteshell - ${project.version} - - - * - * - - - - - - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index 3347297a46..ea334125d5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -145,7 +145,7 @@ public class ParameterUtils { } else if (dataType.equals(DataType.DATE)) { stmt.setDate(index, java.sql.Date.valueOf(value)); } else if (dataType.equals(DataType.TIME)) { - stmt.setTime(index, java.sql.Time.valueOf(value)); + stmt.setString(index, value); } else if (dataType.equals(DataType.TIMESTAMP)) { stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); } else if (dataType.equals(DataType.BOOLEAN)) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertyUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertyUtils.java deleted file mode 100644 index 41c115ab0f..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertyUtils.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.utils; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; - -/** - * property utils - */ -public class PropertyUtils { - - private PropertyUtils() { - throw new IllegalStateException("PropertyUtils class"); - } - - /** - * startParams transform propertyList - * - * @param startParams startParams - * @return startParamList - */ - public static List startParamsTransformPropertyList(String startParams) { - List startParamList = null; - if (startParams != null) { - JsonElement jsonElement = JsonParser.parseString(startParams); - boolean isJson = jsonElement.isJsonObject(); - if (isJson) { - Map startParamMap = JSONUtils.toMap(startParams); - startParamList = startParamMap.entrySet().stream() - .map(entry -> new Property(entry.getKey(), Direct.IN, DataType.VARCHAR, entry.getValue())) - .collect(Collectors.toList()); - } else { - startParamList = JSONUtils.toList(startParams, Property.class); - } - } - return startParamList; - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties index 40e1c5abcb..402112263f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties @@ -26,6 +26,17 @@ resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path=/dolphinscheduler +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=minioadmin +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=minioadmin +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=cn-north-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://localhost:9000 + # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml index 5ec1ce5766..b3e1edb79f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml @@ -31,18 +31,22 @@ org.apache.dolphinscheduler dolphinscheduler-spi + provided org.apache.dolphinscheduler dolphinscheduler-task-api + provided + - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication + software.amazon.awssdk + datasync org.apache.dolphinscheduler dolphinscheduler-common + provided diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java index aff7ba2558..a9f855503a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java @@ -17,9 +17,13 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import org.apache.dolphinscheduler.authentication.aws.DataSyncClientFactory; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.datasync.DataSyncClient; import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionRequest; import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionResponse; @@ -44,7 +48,6 @@ import org.apache.commons.lang3.StringUtils; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import lombok.Data; @@ -70,8 +73,16 @@ public class DatasyncHook { } protected static DataSyncClient createClient() { - Map awsProperties = PropertyUtils.getByPrefix("aws.datasync.", ""); - return DataSyncClientFactory.createDataSyncClient(awsProperties); + final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID); + final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY); + final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION); + + final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey); + final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials); + + // create a datasync client + return DataSyncClient.builder().region(Region.of(awsRegion)).credentialsProvider(awsCredentialsProvider) + .build(); } public Boolean createDatasyncTask(DatasyncParameters parameters) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml index 4a2be06b47..8a3bb85413 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml @@ -25,23 +25,27 @@ dolphinscheduler-task-dms + jar org.apache.dolphinscheduler dolphinscheduler-spi + provided org.apache.dolphinscheduler dolphinscheduler-task-api + provided org.apache.dolphinscheduler dolphinscheduler-common + provided - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication + com.amazonaws + aws-java-sdk-dms diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java index cf2306abb1..40aa6a527d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.task.dms; -import org.apache.dolphinscheduler.authentication.aws.AWSDatabaseMigrationServiceClientFactory; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.commons.io.IOUtils; @@ -29,7 +29,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Date; import java.util.List; -import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; @@ -38,7 +37,11 @@ import lombok.NoArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService; +import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder; import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskRequest; import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskResult; import com.amazonaws.services.databasemigrationservice.model.DeleteReplicationTaskRequest; @@ -84,8 +87,17 @@ public class DmsHook { } public static AWSDatabaseMigrationService createClient() { - Map awsProperties = PropertyUtils.getByPrefix("aws.dms.", ""); - return AWSDatabaseMigrationServiceClientFactory.createAWSDatabaseMigrationServiceClient(awsProperties); + final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID); + final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY); + final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION); + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); + final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); + + // create a DMS client + return AWSDatabaseMigrationServiceClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion) + .build(); } public Boolean createReplicationTask() throws Exception { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml index fa7e51062e..969756a991 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml @@ -28,20 +28,19 @@ jar - org.apache.dolphinscheduler dolphinscheduler-spi + provided - org.apache.dolphinscheduler dolphinscheduler-task-api + provided - - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication + com.amazonaws + aws-java-sdk-emr diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java index 412b0b86e8..6f6ec63a29 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java @@ -22,19 +22,22 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; -import org.apache.dolphinscheduler.authentication.aws.AmazonElasticMapReduceClientFactory; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import java.util.Map; import java.util.TimeZone; import lombok.extern.slf4j.Slf4j; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; +import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; @@ -91,8 +94,22 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask { return emrParameters; } + /** + * create emr client from BasicAWSCredentials + * + * @return AmazonElasticMapReduce + */ protected AmazonElasticMapReduce createEmrClient() { - Map awsProperties = PropertyUtils.getByPrefix("aws.emr.", ""); - return AmazonElasticMapReduceClientFactory.createAmazonElasticMapReduceClient(awsProperties); + + final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID); + final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY); + final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION); + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); + final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); + // create an EMR client + return AmazonElasticMapReduceClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion) + .build(); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/pom.xml index 030ab3df58..7caaf28641 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/pom.xml @@ -31,23 +31,26 @@ org.apache.dolphinscheduler dolphinscheduler-spi + provided org.apache.dolphinscheduler dolphinscheduler-task-api + provided org.apache.dolphinscheduler dolphinscheduler-common + provided + + + com.amazonaws + aws-java-sdk-sagemaker org.apache.dolphinscheduler dolphinscheduler-datasource-all - - - - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication + ${project.version} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java index 595655c891..b1b2cc811f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java @@ -22,9 +22,7 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; -import org.apache.dolphinscheduler.authentication.aws.AmazonSageMakerClientFactory; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.datasource.sagemaker.param.SagemakerConnectionParam; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; @@ -43,7 +41,11 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.sagemaker.AmazonSageMaker; +import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder; import com.amazonaws.services.sagemaker.model.StartPipelineExecutionRequest; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; @@ -184,8 +186,16 @@ public class SagemakerTask extends AbstractRemoteTask { } protected AmazonSageMaker createClient() { - Map awsProperties = PropertyUtils.getByPrefix("aws.sagemaker.", ""); - return AmazonSageMakerClientFactory.createAmazonSageMakerClient(awsProperties); + final String awsAccessKeyId = parameters.getUsername(); + final String awsSecretAccessKey = parameters.getPassword(); + final String awsRegion = parameters.getAwsRegion(); + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); + final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); + // create a SageMaker client + return AmazonSageMakerClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion) + .build(); } } diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index ec7635e4d5..f4c1573226 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -29,7 +29,6 @@ dolphinscheduler-task-all - dolphinscheduler-task-all-prune dolphinscheduler-task-api dolphinscheduler-task-shell dolphinscheduler-task-datax diff --git a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml index 7b206da233..49a95d25d0 100644 --- a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml +++ b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml @@ -38,12 +38,6 @@ 0755 0755 - - ${basedir}/src/main/dist-bin - dist-bin - 0755 - 0755 - ${basedir}/../dolphinscheduler-dao/src/main/resources diff --git a/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh b/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh deleted file mode 100644 index 0be4cd3a11..0000000000 --- a/dolphinscheduler-tools/src/main/dist-bin/create-demo-processes.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} - -if [ "$DOCKER" != "true" ]; then - source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" -fi - -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} - -# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt -CP=$DOLPHINSCHEDULER_HOME/tools/libs/* -for d in api-server; do - for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do - JAR_FILE_NAME=${f##*/} - if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then - CP=$CP:$f - fi - done -done - -$JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$CP":"$DOLPHINSCHEDULER_HOME/tools/sql" \ - -Dspring.profiles.active=demo,${DATABASE} \ - org.apache.dolphinscheduler.tools.demo.CreateProcessDemo diff --git a/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh b/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh deleted file mode 100644 index d4422d763e..0000000000 --- a/dolphinscheduler-tools/src/main/dist-bin/migrate-resource.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} - -if [ "$DOCKER" != "true" ]; then - source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" -fi - -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} - -# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt -CP=$DOLPHINSCHEDULER_HOME/tools/libs/* -for d in api-server; do - for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do - JAR_FILE_NAME=${f##*/} - if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then - CP=$CP:$f - fi - done -done - -$JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$CP":"$DOLPHINSCHEDULER_HOME/tools/sql" \ - -Dspring.profiles.active=resource,${DATABASE} \ - org.apache.dolphinscheduler.tools.resource.MigrateResource $1 diff --git a/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh b/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh deleted file mode 100755 index 4f768ca7b2..0000000000 --- a/dolphinscheduler-tools/src/main/dist-bin/upgrade-schema.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} - -if [ "$DOCKER" != "true" ]; then - source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" -fi - -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} - -# TODO temp solution to may our tarball small enough to pass ASF release policy, for more detail see: https://lists.apache.org/thread/rmp7fghlj0n7h9y2v3p8gkw9f9qbo6qt -CP=$DOLPHINSCHEDULER_HOME/tools/libs/* -for d in api-server; do - for f in $DOLPHINSCHEDULER_HOME/$d/libs/*.jar; do - JAR_FILE_NAME=${f##*/} - if [[ ! $CP =~ $JAR_FILE_NAME ]] && [[ ! $JAR_FILE_NAME =~ "dolphinscheduler-" ]] && [[ ! $JAR_FILE_NAME == "spring"* ]]; then - CP=$CP:$f - fi - done -done - -$JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/sql":"$CP" \ - -Dspring.profiles.active=upgrade,${DATABASE} \ - org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 68eb7b3102..5e5ee4b2a9 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -930,14 +930,12 @@ export default { code: 'Parameter Code', name: 'Parameter Name', value: 'Parameter Value', - data_type: 'Parameter Data Type', create_user: 'Create User', modify_user: 'Modify User', create_time: 'Create Time', update_time: 'Update Time', name_tips: 'Please enter your parameter name', value_tips: 'Please enter your parameter value', - data_type_tips: 'Please choose your parameter data type', operation: 'Operation', edit: 'Edit', delete: 'Delete', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index e1bfd6f672..85de0122fb 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -900,14 +900,12 @@ export default { code: '参数编码', name: '参数名称', value: '参数值', - data_type: '参数类型', create_user: '创建用户', modify_user: '修改用户', create_time: '创建时间', update_time: '更新时间', name_tips: '请输入参数名称', value_tips: '请输入参数值', - data_type_tips: '请选择参数类型', operation: '操作', edit: '编辑', delete: '删除', diff --git a/dolphinscheduler-ui/src/service/modules/projects-parameter/types.ts b/dolphinscheduler-ui/src/service/modules/projects-parameter/types.ts index 0e2a9c76bb..9e0f747136 100644 --- a/dolphinscheduler-ui/src/service/modules/projects-parameter/types.ts +++ b/dolphinscheduler-ui/src/service/modules/projects-parameter/types.ts @@ -19,7 +19,6 @@ interface ListReq { pageNo: number pageSize: number searchVal?: string - projectParameterDataType?: string } interface ProjectParameterCodeReq { @@ -29,7 +28,6 @@ interface ProjectParameterCodeReq { interface ProjectParameterReq { projectParameterName: string projectParameterValue: string - projectParameterDataType: string } interface UpdateProjectParameterReq extends ProjectParameterReq { @@ -41,7 +39,6 @@ interface ProjectParameterList { code: number name: string value: string - data_type: string createTime: string updateTime: string } diff --git a/dolphinscheduler-ui/src/views/projects/parameter/components/parameter-modal.tsx b/dolphinscheduler-ui/src/views/projects/parameter/components/parameter-modal.tsx index d0e20569d1..c69ae864b2 100644 --- a/dolphinscheduler-ui/src/views/projects/parameter/components/parameter-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/parameter/components/parameter-modal.tsx @@ -23,10 +23,9 @@ import { watch } from 'vue' import Modal from '@/components/modal' -import { NForm, NFormItem, NInput, NSelect } from 'naive-ui' +import { NForm, NFormItem, NInput } from 'naive-ui' import { useModal } from './use-modal' import { useI18n } from 'vue-i18n' -import { DATA_TYPES_MAP, DEFAULT_DATA_TYPE } from "@/views/projects/parameter/data_type" const ParameterModal = defineComponent({ name: 'ParameterModal', @@ -53,11 +52,9 @@ const ParameterModal = defineComponent({ if (props.statusRef === 0) { variables.model.projectParameterName = '' variables.model.projectParameterValue = '' - variables.model.projectParameterDataType = DEFAULT_DATA_TYPE } else { variables.model.projectParameterName = props.row.paramName variables.model.projectParameterValue = props.row.paramValue - variables.model.projectParameterDataType = props.row.paramDataType } ctx.emit('cancelModal', props.showModalRef) } @@ -81,12 +78,10 @@ const ParameterModal = defineComponent({ if (props.statusRef === 0) { variables.model.projectParameterName = '' variables.model.projectParameterValue = '' - variables.model.projectParameterDataType = DEFAULT_DATA_TYPE } else { variables.model.code = props.row.code variables.model.projectParameterName = props.row.paramName variables.model.projectParameterValue = props.row.paramValue - variables.model.projectParameterDataType = props.row.paramDataType } } ) @@ -97,7 +92,6 @@ const ParameterModal = defineComponent({ variables.model.code = props.row.code variables.model.projectParameterName = props.row.paramName variables.model.projectParameterValue = props.row.paramValue - variables.model.projectParameterDataType = props.row.paramDataType } ) @@ -118,8 +112,7 @@ const ParameterModal = defineComponent({ onConfirm={this.confirmModal} confirmDisabled={ !this.model.projectParameterName || - !this.model.projectParameterValue || - !this.model.projectParameterDataType + !this.model.projectParameterValue } confirmClassName='btn-submit' cancelClassName='btn-cancel' @@ -142,15 +135,6 @@ const ParameterModal = defineComponent({ v-model={[this.model.projectParameterValue, 'value']} /> - - { - return { value: item, label: item } - })} - v-model={[this.model.projectParameterDataType, 'value']} - /> - ) }} diff --git a/dolphinscheduler-ui/src/views/projects/parameter/components/use-modal.ts b/dolphinscheduler-ui/src/views/projects/parameter/components/use-modal.ts index 50dae38a49..56b5186846 100644 --- a/dolphinscheduler-ui/src/views/projects/parameter/components/use-modal.ts +++ b/dolphinscheduler-ui/src/views/projects/parameter/components/use-modal.ts @@ -27,7 +27,6 @@ import { UpdateProjectParameterReq } from '@/service/modules/projects-parameter/types' import { useRouter } from 'vue-router' -import { DEFAULT_DATA_TYPE } from "@/views/projects/parameter/data_type"; export function useModal( props: any, @@ -42,8 +41,7 @@ export function useModal( model: { code: ref(-1), projectParameterName: ref(''), - projectParameterValue: ref(''), - projectParameterDataType: ref(DEFAULT_DATA_TYPE) + projectParameterValue: ref('') }, saving: false, rules: { @@ -64,15 +62,6 @@ export function useModal( return new Error(t('project.parameter.value_tips')) } } - }, - data_type: { - required: true, - trigger: ['input', 'blur'], - validator() { - if (variables.model.projectParameterDataType === '') { - return new Error(t('project.parameter.data_type_tips')) - } - } } } }) @@ -94,14 +83,12 @@ export function useModal( const submitModal = () => { const data: ProjectParameterReq = { projectParameterName: variables.model.projectParameterName, - projectParameterValue: variables.model.projectParameterValue, - projectParameterDataType: variables.model.projectParameterDataType + projectParameterValue: variables.model.projectParameterValue } createProjectParameter(data, variables.projectCode).then(() => { variables.model.projectParameterName = '' variables.model.projectParameterValue = '' - variables.model.projectParameterDataType = DEFAULT_DATA_TYPE ctx.emit('confirmModal', props.showModalRef) }) } @@ -110,8 +97,7 @@ export function useModal( const data: UpdateProjectParameterReq = { code: variables.model.code, projectParameterName: variables.model.projectParameterName, - projectParameterValue: variables.model.projectParameterValue, - projectParameterDataType: variables.model.projectParameterDataType + projectParameterValue: variables.model.projectParameterValue } updateProjectParameter(data, variables.projectCode).then(() => { diff --git a/dolphinscheduler-ui/src/views/projects/parameter/data_type.ts b/dolphinscheduler-ui/src/views/projects/parameter/data_type.ts deleted file mode 100644 index 1d90384436..0000000000 --- a/dolphinscheduler-ui/src/views/projects/parameter/data_type.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export const DATA_TYPES_MAP = { - VARCHAR: { - alias: 'VARCHAR' - }, - INTEGER: { - alias: 'INTEGER' - }, - LONG: { - alias: 'LONG' - }, - FLOAT: { - alias: 'FLOAT' - }, - DOUBLE: { - alias: 'DOUBLE' - }, - DATE: { - alias: 'DATE' - }, - TIME: { - alias: 'TIME' - }, - TIMESTAMP: { - alias: 'TIMESTAMP' - }, - BOOLEAN: { - alias: 'BOOLEAN' - }, - LIST: { - alias: 'LIST' - }, - FILE: { - alias: 'FILE' - } -} - -export const DEFAULT_DATA_TYPE = 'VARCHAR' diff --git a/dolphinscheduler-ui/src/views/projects/parameter/index.tsx b/dolphinscheduler-ui/src/views/projects/parameter/index.tsx index 6b9519830b..f8d0719f18 100644 --- a/dolphinscheduler-ui/src/views/projects/parameter/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/parameter/index.tsx @@ -21,8 +21,7 @@ import { NInput, NPagination, NSpace, - NButton, - NSelect + NButton } from 'naive-ui' import { defineComponent, onMounted, toRefs, watch } from 'vue' import { useI18n } from 'vue-i18n' @@ -30,7 +29,6 @@ import { useTable } from '@/views/projects/parameter/use-table' import Card from '@/components/card' import ParameterModal from '@/views/projects/parameter/components/parameter-modal' import { SearchOutlined } from '@vicons/antd' -import { DATA_TYPES_MAP } from "@/views/projects/parameter/data_type" export default defineComponent({ name: 'ProjectParameterList', @@ -42,7 +40,6 @@ export default defineComponent({ pageSize: variables.pageSize, pageNo: variables.page, searchVal: variables.searchVal, - projectParameterDataType: variables.projectParameterDataType, projectCode: variables.projectCode }) } @@ -119,16 +116,6 @@ export default defineComponent({ v-model={[this.searchVal, 'value']} placeholder={t('project.parameter.name')} /> - { - return { value: item, label: item } - })} - placeholder={t('project.parameter.data_type_tips')} - style={{ width: '180px' }} - clearable - /> diff --git a/dolphinscheduler-ui/src/views/projects/parameter/use-table.ts b/dolphinscheduler-ui/src/views/projects/parameter/use-table.ts index 6d98a7e163..9d0bf35dcd 100644 --- a/dolphinscheduler-ui/src/views/projects/parameter/use-table.ts +++ b/dolphinscheduler-ui/src/views/projects/parameter/use-table.ts @@ -45,7 +45,6 @@ export function useTable() { page: ref(1), pageSize: ref(10), searchVal: ref(), - projectParameterDataType: ref(), totalPage: ref(1), showRef: ref(false), statusRef: ref(0), @@ -70,11 +69,6 @@ export function useTable() { key: 'paramValue', ...COLUMN_WIDTH_CONFIG['name'] }, - { - title: t('project.parameter.data_type'), - key: 'paramDataType', - ...COLUMN_WIDTH_CONFIG['name'] - }, { title: t('project.parameter.create_user'), key: 'createUser', @@ -198,8 +192,7 @@ export function useTable() { getTableData({ pageSize: variables.pageSize, pageNo: variables.page, - searchVal: variables.searchVal, - projectParameterDataType: variables.projectParameterDataType + searchVal: variables.searchVal }) }) } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx index bfd76e79b1..9a55f8d3b2 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx @@ -154,8 +154,7 @@ export default defineComponent({ formValue.value.globalParams = process.globalParamList.map((param) => ({ key: param.prop, value: param.value, - direct: param.direct, - type: param.type + direct: param.direct })) } } @@ -240,7 +239,6 @@ export default defineComponent({ return { key: '', direct: 'IN', - type: 'VARCHAR', value: '' } }} @@ -248,16 +246,16 @@ export default defineComponent({ > {{ default: (param: { - value: { key: string; direct: string; type: string; value: string } + value: { key: string; direct: string; value: string } }) => ( - + - + - - - - + - { - return { - key: '', - direct: 'IN', - type: 'VARCHAR', - value: '' - } - }} - class='input-startup-params' - > - {{ - default: (param: { - value: { prop: string; direct: string; type: string; value: string } - }) => ( - - - - - - - - - - - - - - - ) - }} - + {this.startParamsList.length === 0 ? ( + + + + + + ) : ( + + {this.startParamsList.map((item, index) => ( + + + this.updateParamsList(index, param) + } + /> + this.removeStartParams(index)} + class='btn-delete-custom-parameter' + > + + + + + + + + + + + ))} + + )} conf - - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources - - **/*.yaml - - conf - diff --git a/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh index 95b283f91e..42f8b1c656 100644 --- a/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh @@ -24,7 +24,6 @@ -XX:+PrintGCDetails -Xloggc:gc.log --XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorHolder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorHolder.java index e44b23152c..fa07dbfde6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorHolder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorHolder.java @@ -48,10 +48,6 @@ public class WorkerTaskExecutorHolder { workerTaskExecutorMap.clear(); } - public static int size() { - return workerTaskExecutorMap.size(); - } - public static Collection getAllTaskExecutor() { return workerTaskExecutorMap.values(); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPool.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPool.java index 99645f89a4..4606d9f7e9 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPool.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPool.java @@ -39,7 +39,6 @@ public class WorkerTaskExecutorThreadPool { public WorkerTaskExecutorThreadPool(WorkerConfig workerConfig) { this.threadPoolExecutor = ThreadUtils.newDaemonFixedThreadExecutor("WorkerTaskExecutorThreadPool", workerConfig.getExecThreads()); - threadPoolExecutor.prestartAllCoreThreads(); this.workerConfig = workerConfig; WorkerServerMetrics.registerWorkerExecuteQueueSizeGauge(this::getWaitingTaskExecutorSize); @@ -65,19 +64,15 @@ public class WorkerTaskExecutorThreadPool { } public boolean isOverload() { - return WorkerTaskExecutorHolder.size() >= workerConfig.getExecThreads(); + return threadPoolExecutor.getQueue().size() > 0; } public int getWaitingTaskExecutorSize() { - if (WorkerTaskExecutorHolder.size() <= workerConfig.getExecThreads()) { - return 0; - } else { - return WorkerTaskExecutorHolder.size() - workerConfig.getExecThreads(); - } + return threadPoolExecutor.getQueue().size(); } public int getRunningTaskExecutorSize() { - return Math.min(WorkerTaskExecutorHolder.size(), workerConfig.getExecThreads()); + return threadPoolExecutor.getActiveCount(); } /** diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java index dc1ea8b2e5..d55765d23f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java @@ -89,8 +89,8 @@ public class TaskInstanceKillOperationFunction taskExecutionContext .setCurrentExecutionStatus(result ? TaskExecutionStatus.SUCCESS : TaskExecutionStatus.FAILURE); - WorkerTaskExecutorHolder.remove(taskInstanceId); - messageRetryRunner.removeRetryMessages(taskInstanceId); + WorkerTaskExecutorHolder.remove(taskExecutionContext.getTaskInstanceId()); + messageRetryRunner.removeRetryMessages(taskExecutionContext.getTaskInstanceId()); return TaskInstanceKillResponse.success(taskExecutionContext); } finally { LogUtils.removeTaskInstanceIdMDC(); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java index 9f27ac4309..182ac6a1c2 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorThreadPoolTest.java @@ -17,127 +17,48 @@ package org.apache.dolphinscheduler.server.worker.runner; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.TaskExecuteThreadsFullPolicy; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; +import org.apache.dolphinscheduler.spi.enums.ResourceType; -import org.apache.commons.lang3.RandomUtils; +import java.io.IOException; +import java.util.List; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import com.google.common.truth.Truth; - class WorkerTaskExecutorThreadPoolTest { - @BeforeEach - public void setUp() { - WorkerTaskExecutorHolder.clear(); - } - @Test public void testIsOverload() { - final int execThreadCount = RandomUtils.nextInt(1, 100); - final int totalTaskCount = RandomUtils.nextInt(1, 10000); - final WorkerConfig workerConfig = createWorkerConfig(execThreadCount, TaskExecuteThreadsFullPolicy.CONTINUE); - final WorkerTaskExecutorThreadPool workerTaskExecutorThreadPool = - new WorkerTaskExecutorThreadPool(workerConfig); - // submit totalTaskCount task, the thread pool size is execThreadCount, reject policy is CONTINUE - // after submit execThreadCount task, the thread pool is overload - for (int i = 1; i <= totalTaskCount; i++) { - MockWorkerTaskExecutor mockWorkerTaskExecutor = - new MockWorkerTaskExecutor(() -> ThreadUtils.sleep(10_000L)); - workerTaskExecutorThreadPool.submitWorkerTaskExecutor(mockWorkerTaskExecutor); - if (i >= execThreadCount) { - Truth.assertThat(workerTaskExecutorThreadPool.isOverload()).isTrue(); - } else { - Truth.assertThat(workerTaskExecutorThreadPool.isOverload()).isFalse(); - } - } - } - - @Test - public void testSubmitWorkerTaskExecutorWithContinuePolicy() { - final int execThreadCount = RandomUtils.nextInt(1, 100); - final int totalTaskCount = RandomUtils.nextInt(1, 10000); - final WorkerConfig workerConfig = createWorkerConfig(execThreadCount, TaskExecuteThreadsFullPolicy.CONTINUE); - final WorkerTaskExecutorThreadPool workerTaskExecutorThreadPool = - new WorkerTaskExecutorThreadPool(workerConfig); - // submit totalTaskCount task, the thread pool size is execThreadCount, reject policy is CONTINUE - // all task will be submitted success - for (int i = 1; i <= totalTaskCount; i++) { - MockWorkerTaskExecutor mockWorkerTaskExecutor = - new MockWorkerTaskExecutor(() -> ThreadUtils.sleep(10_000L)); - Truth.assertThat(workerTaskExecutorThreadPool.submitWorkerTaskExecutor(mockWorkerTaskExecutor)).isTrue(); - } - } - - @Test - public void testSubmitWorkerTaskExecutorWithRejectPolicy() { - final int execThreadCount = RandomUtils.nextInt(1, 100); - final int totalTaskCount = RandomUtils.nextInt(1, 10000); - final WorkerConfig workerConfig = createWorkerConfig(execThreadCount, TaskExecuteThreadsFullPolicy.REJECT); - final WorkerTaskExecutorThreadPool workerTaskExecutorThreadPool = - new WorkerTaskExecutorThreadPool(workerConfig); - // submit totalTaskCount task, the thread pool size is execThreadCount, reject policy is REJECT - // only the front execThreadCount task will be submitted success - for (int i = 1; i <= totalTaskCount; i++) { - MockWorkerTaskExecutor mockWorkerTaskExecutor = - new MockWorkerTaskExecutor(() -> ThreadUtils.sleep(10_000L)); - boolean submitResult = workerTaskExecutorThreadPool.submitWorkerTaskExecutor(mockWorkerTaskExecutor); - if (i <= execThreadCount) { - Assertions.assertTrue(submitResult, "The " + i + " task should submit success"); - } else { - Assertions.assertFalse(submitResult, "The " + i + " task should submit failed"); - } - } - } - - @Test - public void testGetWaitingTaskExecutorSize() { - final int execThreadCount = RandomUtils.nextInt(1, 100); - final int totalTaskCount = RandomUtils.nextInt(1, 10000); - final WorkerConfig workerConfig = createWorkerConfig(execThreadCount, TaskExecuteThreadsFullPolicy.CONTINUE); - final WorkerTaskExecutorThreadPool workerTaskExecutorThreadPool = - new WorkerTaskExecutorThreadPool(workerConfig); - - Truth.assertThat(workerTaskExecutorThreadPool.getWaitingTaskExecutorSize()).isEqualTo(0); - for (int i = 1; i <= totalTaskCount; i++) { - MockWorkerTaskExecutor mockWorkerTaskExecutor = - new MockWorkerTaskExecutor(() -> ThreadUtils.sleep(10_000L)); - workerTaskExecutorThreadPool.submitWorkerTaskExecutor(mockWorkerTaskExecutor); - if (i <= execThreadCount) { - Truth.assertThat(workerTaskExecutorThreadPool.getWaitingTaskExecutorSize()).isEqualTo(0); - } else { - Truth.assertThat(workerTaskExecutorThreadPool.getWaitingTaskExecutorSize()) - .isEqualTo(i - execThreadCount); - } - } - } - - @Test - public void testGetRunningTaskExecutorSize() { - final int execThreadCount = RandomUtils.nextInt(1, 100); - final int totalTaskCount = RandomUtils.nextInt(1, 10000); - WorkerConfig workerConfig = createWorkerConfig(execThreadCount, TaskExecuteThreadsFullPolicy.CONTINUE); + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setExecThreads(1); + workerConfig.setTaskExecuteThreadsFullPolicy(TaskExecuteThreadsFullPolicy.CONTINUE); WorkerTaskExecutorThreadPool workerTaskExecutorThreadPool = new WorkerTaskExecutorThreadPool(workerConfig); - - Truth.assertThat(workerTaskExecutorThreadPool.getRunningTaskExecutorSize()).isEqualTo(0); - for (int i = 1; i <= totalTaskCount; i++) { - MockWorkerTaskExecutor mockWorkerTaskExecutor = - new MockWorkerTaskExecutor(() -> ThreadUtils.sleep(10_000L)); - workerTaskExecutorThreadPool.submitWorkerTaskExecutor(mockWorkerTaskExecutor); - if (i <= execThreadCount) { - Truth.assertThat(workerTaskExecutorThreadPool.getRunningTaskExecutorSize()).isEqualTo(i); - } else { - Truth.assertThat(workerTaskExecutorThreadPool.getRunningTaskExecutorSize()).isEqualTo(execThreadCount); - } + // submit 100 task, the thread pool size is 1 + // assert the overload should be true + // assert the submitQueue should be 99 + for (int i = 0; i < 100; i++) { + boolean submitResult = + workerTaskExecutorThreadPool.submitWorkerTaskExecutor(new MockWorkerTaskExecutor(() -> { + try { + Thread.sleep(10_000L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + })); + Assertions.assertTrue(submitResult); } + Assertions.assertTrue(workerTaskExecutorThreadPool.isOverload()); + Assertions.assertEquals(99, workerTaskExecutorThreadPool.getWaitingTaskExecutorSize()); + Assertions.assertEquals(1, workerTaskExecutorThreadPool.getRunningTaskExecutorSize()); } static class MockWorkerTaskExecutor extends WorkerTaskExecutor { @@ -145,11 +66,116 @@ class WorkerTaskExecutorThreadPoolTest { private final Runnable runnable; protected MockWorkerTaskExecutor(Runnable runnable) { - super(TaskExecutionContext.builder().taskInstanceId((int) System.nanoTime()).build(), - new WorkerConfig(), - new WorkerMessageSender(), - null, - new WorkerRegistryClient()); + super(TaskExecutionContext.builder().taskInstanceId((int) System.nanoTime()).build(), new WorkerConfig(), + new WorkerMessageSender(), new StorageOperate() { + + @Override + public void createTenantDirIfNotExists(String tenantCode) { + + } + + @Override + public String getResDir(String tenantCode) { + return null; + } + + @Override + public String getUdfDir(String tenantCode) { + return null; + } + + @Override + public boolean mkdir(String tenantCode, String path) throws IOException { + return false; + } + + @Override + public String getResourceFullName(String tenantCode, String fileName) { + return null; + } + + @Override + public String getResourceFileName(String tenantCode, String fullName) { + return null; + } + + @Override + public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { + return null; + } + + @Override + public boolean exists(String fullName) { + return false; + } + + @Override + public boolean delete(String filePath, boolean recursive) { + return false; + } + + @Override + public boolean delete(String filePath, List childrenPathArray, + boolean recursive) { + return false; + } + + @Override + public boolean copy(String srcPath, String dstPath, boolean deleteSource, + boolean overwrite) { + return false; + } + + @Override + public String getDir(ResourceType resourceType, String tenantCode) { + return null; + } + + @Override + public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, + boolean overwrite) { + return false; + } + + @Override + public void download(String srcFilePath, String dstFile, boolean overwrite) { + + } + + @Override + public List vimFile(String tenantCode, String filePath, int skipLineNums, + int limit) { + return null; + } + + @Override + public void deleteTenant(String tenantCode) { + + } + + @Override + public ResUploadType returnStorageType() { + return null; + } + + @Override + public List listFilesStatusRecursively(String path, String defaultPath, + String tenantCode, ResourceType type) { + return null; + } + + @Override + public List listFilesStatus(String path, String defaultPath, String tenantCode, + ResourceType type) throws Exception { + return null; + } + + @Override + public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, + ResourceType type) throws Exception { + return null; + } + }, new WorkerRegistryClient()); this.runnable = runnable; } @@ -164,12 +190,4 @@ class WorkerTaskExecutorThreadPoolTest { } } - private WorkerConfig createWorkerConfig(int execThreads, - TaskExecuteThreadsFullPolicy taskExecuteThreadsFullPolicy) { - WorkerConfig workerConfig = new WorkerConfig(); - workerConfig.setExecThreads(execThreads); - workerConfig.setTaskExecuteThreadsFullPolicy(taskExecuteThreadsFullPolicy); - return workerConfig; - } - } diff --git a/pom.xml b/pom.xml index ab3ba9404e..fc5e994bf9 100755 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,6 @@ dolphinscheduler-storage-plugin dolphinscheduler-extract dolphinscheduler-dao-plugin - dolphinscheduler-authentication @@ -233,11 +232,6 @@ dolphinscheduler-task-all ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-task-all-prune - ${project.version} - org.apache.dolphinscheduler @@ -300,12 +294,6 @@ ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-aws-authentication - ${project.version} - - org.junit junit-bom diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 3c329340e7..33987119db 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -11,6 +11,7 @@ annotations-13.0.jar apache-client-2.17.282.jar asm-9.6.jar aspectjweaver-1.9.7.jar +aspectjrt-1.9.7.jar auth-2.17.282.jar audience-annotations-0.12.0.jar avro-1.7.7.jar @@ -434,6 +435,8 @@ azure-core-management-1.10.1.jar api-common-2.6.0.jar auto-value-1.10.1.jar auto-value-annotations-1.10.4.jar +bcpkix-jdk15on-1.67.jar +bcprov-jdk15on-1.67.jar conscrypt-openjdk-uber-2.5.2.jar gapic-google-cloud-storage-v2-2.18.0-alpha.jar gax-2.23.0.jar