diff --git a/.asf.yaml b/.asf.yaml index 84619447be..abeb08c0b8 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -46,7 +46,7 @@ github: - E2E - Docs - Frontend Build - - "Mergeable: milestone-label-check" +# - "Mergeable: milestone-label-check" required_pull_request_reviews: dismiss_stale_reviews: true required_approving_review_count: 2 diff --git a/.github/mergeable.yml b/.github/mergeable.yml deleted file mode 100644 index a1df2e7410..0000000000 --- a/.github/mergeable.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -version: 2 -mergeable: - # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643, - # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as - # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like, - # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened - - when: pull_request.opened, pull_request.reopened, pull_request.synchronize - name: sync-sql-ddl - validate: - # Sql files must change synchronize - - do: dependent - files: - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql' - message: 'Sql files not change synchronize' - # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize - fail: - - do: comment - payload: - body: > - :warning: This PR do not change database DDL synchronize. - leave_old_comment: false - - do: labels - add: 'sql not sync' - # Remove labels 'sql not sync' if pass - pass: - - do: labels - delete: 'sql not sync' - - - when: pull_request.* - name: milestone-label-check - validate: - - do: milestone - no_empty: - enabled: false # Cannot be empty when true. - message: 'Milestone is required and cannot be empty.' - - do: label - and: - - must_include: - regex: 'feature|bug|improvement|document|chore|revert' - message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`' - - must_include: - regex: 'ready-to-merge' - message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.' diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 671223d286..37fb1bcd7e 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -153,7 +153,7 @@ jobs: fail-fast: false matrix: db: ["mysql", "postgresql"] - version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"] + version: ["3.1.9", "3.2.0"] steps: - name: Set up JDK 8 uses: actions/setup-java@v2 diff --git a/.github/workflows/mergeable.yml b/.github/workflows/mergeable.yml new file mode 100644 index 0000000000..8b7bd8799c --- /dev/null +++ b/.github/workflows/mergeable.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +#version: 2 + +on: + pull_request: + +name: "Mergeable" + +jobs: + result: + name: "Mergeable: milestone-label-check" + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Status + run: | + echo "Temporary skipping this check" + +#mergeable: +# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643, +# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as +# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like, +# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened +# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize +# name: sync-sql-ddl +# validate: +# # Sql files must change synchronize +# - do: dependent +# files: +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql' +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql' +# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql' +# message: 'Sql files not change synchronize' +# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize +# fail: +# - do: comment +# payload: +# body: > +# :warning: This PR do not change database DDL synchronize. +# leave_old_comment: false +# - do: labels +# add: 'sql not sync' +# # Remove labels 'sql not sync' if pass +# pass: +# - do: labels +# delete: 'sql not sync' +# +# - when: pull_request.* +# name: milestone-label-check +# validate: +# - do: milestone +# no_empty: +# enabled: false # Cannot be empty when true. +# message: 'Milestone is required and cannot be empty.' +# - do: label +# and: +# - must_include: +# regex: 'feature|bug|improvement|document|chore|revert' +# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`' +# - must_include: +# regex: 'ready-to-merge' +# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.' diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 6c9f41d7a3..a7e78a11f7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -76,7 +76,7 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Run Unit tests - run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false + run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false - name: Upload coverage report to codecov run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) @@ -99,23 +99,11 @@ jobs: -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 -Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/* -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - -DskipUT=true -DskipIT=true + -DskipUT=true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - name: Collect logs - continue-on-error: true - run: | - mkdir -p ${LOG_DIR} - docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt - - - name: Upload logs - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: unit-test-logs - path: ${LOG_DIR} result: name: Unit Test runs-on: ubuntu-latest diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index ba533b6e47..4b83f5b160 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -120,6 +120,12 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false | | conf.common."alert.rpc.port" | int | `50052` | rpc port | | conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop | +| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | | +| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required | +| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required | +| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. | +| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn | +| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required | | conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh | | conf.common."data-quality.jar.dir" | string | `nil` | data quality option | | conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions | @@ -138,11 +144,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS | | conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS | | conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS | -| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required | -| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required | -| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. | -| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn | -| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required | | conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS | | conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS | | conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS | @@ -158,6 +159,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | conf.common."yarn.application.status.address" | string | `"http://ds1:%s/ws/v1/cluster/apps/%s"` | if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname | | conf.common."yarn.job.history.status.address" | string | `"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"` | job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) | | conf.common."yarn.resourcemanager.ha.rm.ids" | string | `"192.168.xx.xx,192.168.xx.xx"` | if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty | +| datasource.profile | string | `"postgresql"` | The profile of datasource | | externalDatabase.database | string | `"dolphinscheduler"` | The database of external database | | externalDatabase.driverClassName | string | `"org.postgresql.Driver"` | The driverClassName of external database | | externalDatabase.enabled | bool | `false` | If exists external database, and set postgresql.enable value to false. external database will be used, otherwise Dolphinscheduler's internal database will be used. | @@ -306,7 +308,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. | | worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. | | worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. | -| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. | | worker.keda.advanced | object | `{}` | Specify HPA related options | | worker.keda.cooldownPeriod | int | `30` | How many seconds KEDA will wait before scaling to zero. Note that HPA has a separate cooldown period for scale-downs | | worker.keda.enabled | bool | `false` | Enable or disable the Keda component | diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 0b2a542cc1..368e0b290f 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -51,7 +51,6 @@ Create a default common labels. {{- define "dolphinscheduler.common.labels" -}} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/version: {{ .Chart.AppVersion }} {{- end -}} {{/* @@ -146,6 +145,10 @@ Create a database environment variables. {{- else }} value: {{ .Values.externalDatabase.type | quote }} {{- end }} +{{- if or .Values.mysql.enabled (eq .Values.externalDatabase.type "mysql") }} +- name: SPRING_PROFILES_ACTIVE + value: mysql +{{- end }} - name: SPRING_DATASOURCE_URL {{- if .Values.postgresql.enabled }} value: jdbc:postgresql://{{ template "dolphinscheduler.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresqlDatabase }}?{{ .Values.postgresql.params }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml new file mode 100644 index 0000000000..fd89c1ed15 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -0,0 +1,115 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if and .Values.alert.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-alert + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} +data: + application.yaml: | + spring: + profiles: + active: {{ .Values.datasource.profile }} + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + banner: + charset: UTF-8 + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + + # Mybatis-plus configuration, you don't need to change it + mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + server: + port: 50053 + + management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + + alert: + port: 50052 + # Mark each alert of alert server if late after x milliseconds as failed. + # Define value is (0 = infinite), and alert server would be waiting alert result. + wait-timeout: 0 + max-heartbeat-interval: 60s + query_alert_threshold: 100 + + registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + + metrics: + enabled: true + + # Override by profile + --- + spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml new file mode 100644 index 0000000000..f115490c31 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml @@ -0,0 +1,258 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if and .Values.api.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-api + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + {{- include "dolphinscheduler.api.labels" . | nindent 4 }} +data: + application.yaml: | + server: + port: 12345 + servlet: + session: + timeout: 120m + context-path: /dolphinscheduler/ + compression: + enabled: true + mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml + jetty: + max-http-form-post-size: 5000000 + accesslog: + enabled: true + custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms' + + spring: + profiles: + active: {{ .Values.datasource.profile }} + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB + messages: + basename: i18n/messages + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + auto-startup: false + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + mvc: + pathmatch: + matching-strategy: ANT_PATH_MATCHER + static-path-pattern: /static/** + springdoc: + swagger-ui: + path: /swagger-ui.html + packages-to-scan: org.apache.dolphinscheduler.api + + # Mybatis-plus configuration, you don't need to change it + mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + + registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + + api: + audit-enable: false + # Traffic control, if you turn on this config, the maximum number of request/s will be limited. + # global max request number per second + # default tenant-level max request number + traffic-control: + global-switch: false + max-global-qps-rate: 300 + tenant-switch: false + default-tenant-qps-rate: 10 + #customize-tenant-qps-rate: + # eg. + #tenant1: 11 + #tenant2: 20 + python-gateway: + # Weather enable python gateway server or not. The default value is false. + enabled: false + # Authentication token for connection from python api to python gateway server. Should be changed the default value + # when you deploy in public network. + auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + + metrics: + enabled: true + + security: + authentication: + # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO) + type: PASSWORD + # IF you set type `LDAP`, below config will be effective + ldap: + # ldap server config + urls: ldap://ldap.forumsys.com:389/ + base-dn: dc=example,dc=com + username: cn=read-only-admin,dc=example,dc=com + password: password + user: + # admin userId when you use LDAP login + admin: read-only-admin + identity-attribute: uid + email-attribute: mail + # action when ldap user is not exist (supported types: CREATE,DENY) + not-exist-action: CREATE + ssl: + enable: false + # jks file absolute path && password + trust-store: "/ldapkeystore.jks" + trust-store-password: "password" + casdoor: + user: + admin: "" + oauth2: + enable: false + provider: + github: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: github + google: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: google + casdoor: + # Your Casdoor server url + endpoint: "" + client-id: "" + client-secret: "" + # The certificate may be multi-line, you can use `|-` for ease + certificate: "" + # Your organization name added in Casdoor + organization-name: "" + # Your application name added in Casdoor + application-name: "" + # Doplhinscheduler login url + redirect-url: "" + + # Override by profile + --- + spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate +{{- end }} + + + diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml new file mode 100644 index 0000000000..0a2c8bf2a4 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -0,0 +1,173 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if and .Values.master.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + {{- include "dolphinscheduler.master.labels" . | nindent 4 }} +data: + application.yaml: | + spring: + profiles: + active: {{ .Values.datasource.profile }} + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.threadPool.threadPriority: 5 + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.threadPool.makeThreadsDaemons: true + org.quartz.threadPool.threadCount: 25 + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + + # Mybatis-plus configuration, you don't need to change it + mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + + registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + + master: + listen-port: 5678 + # master fetch command num + fetch-command-num: 10 + # master prepare execute thread number to limit handle commands in parallel + pre-exec-threads: 10 + # master execute thread number to limit process instances in parallel + exec-threads: 100 + # master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s. + dispatch-task-number: 3 + # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight + host-selector: lower_weight + # master heartbeat interval + max-heartbeat-interval: 10s + # master commit task retry times + task-commit-retry-times: 5 + # master commit task interval + task-commit-interval: 1s + state-wheel-interval: 5s + server-load-protection: + # If set true, will open master overload protection + enabled: true + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. + max-disk-usage-percentage-thresholds: 0.7 + # failover interval, the unit is minute + failover-interval: 10m + # kill yarn / k8s application when failover taskInstance, default true + kill-application-when-task-failover: true + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: waiting + # The max waiting time to reconnect to registry if you set the strategy to waiting + max-waiting-time: 100s + worker-group-refresh-interval: 10s + + server: + port: 5679 + + management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + + metrics: + enabled: true + + # Override by profile + --- + spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml new file mode 100644 index 0000000000..b15cad5649 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if and .Values.worker.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + {{- include "dolphinscheduler.worker.labels" . | nindent 4 }} +data: + application.yaml: | + spring: + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + + registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + + worker: + # worker listener port + listen-port: 1234 + # worker execute thread number to limit task instances in parallel + exec-threads: 100 + # worker heartbeat interval + max-heartbeat-interval: 10s + # worker host weight to dispatch tasks, default value 100 + host-weight: 100 + server-load-protection: + # If set true, will open worker overload protection + enabled: true + # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. + max-disk-usage-percentage-thresholds: 0.7 + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: waiting + # The max waiting time to reconnect to registry if you set the strategy to waiting + max-waiting-time: 100s + task-execute-threads-full-policy: REJECT + tenant-config: + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. + auto-create-tenant-enabled: true + # Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants. + distributed-tenant-enabled: false + # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. + default-tenant-enabled: false + + server: + port: 1235 + + management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + + metrics: + enabled: true +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml index c9af1c00a5..8c1d515ca7 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml @@ -37,7 +37,7 @@ data: {{- range $key, $value := index .Values.conf "common" }} {{- if and $.Values.minio.enabled }} {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }} - {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }} + {{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }} {{- end }} {{ $key }}={{ $value }} {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 54316faeed..6a5560b3d4 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -115,6 +115,9 @@ spec: - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties subPath: common.properties + - name: alert-config-volume + mountPath: /opt/dolphinscheduler/conf/application.yaml + subPath: application.yaml volumes: - name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.alert.persistentVolumeClaim.enabled }} @@ -126,4 +129,7 @@ spec: - name: config-volume configMap: name: {{ include "dolphinscheduler.fullname" . }}-configs + - name: alert-config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-alert {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 47bd4c3ca9..f36d2442ae 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -121,6 +121,9 @@ spec: mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml subPath: task-type-config.yaml {{- end }} + - name: api-config-volume + mountPath: /opt/dolphinscheduler/conf/application.yaml + subPath: application.yaml {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }} @@ -136,6 +139,9 @@ spec: - name: config-volume configMap: name: {{ include "dolphinscheduler.fullname" . }}-configs + - name: api-config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} {{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 66b4f8f8ee..1d9d517d0e 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -109,6 +109,9 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-master + - name: master-config-volume + mountPath: /opt/dolphinscheduler/conf/application.yaml + subPath: application.yaml {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties @@ -122,6 +125,9 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: master-config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} - name: config-volume configMap: diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 231f943fa6..8d55be6f70 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -111,6 +111,9 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + - name: worker-config-volume + mountPath: /opt/dolphinscheduler/conf/application.yaml + subPath: application.yaml - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties subPath: common.properties @@ -139,6 +142,9 @@ spec: - name: {{ include "dolphinscheduler.fullname" . }}-worker-logs emptyDir: {} {{- end }} + - name: worker-config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-worker - name: config-volume configMap: name: {{ include "dolphinscheduler.fullname" . }}-configs diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 7a04ff5604..e7e42c6d09 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -49,6 +49,10 @@ image: # -- tools image tools: dolphinscheduler-tools +datasource: + # -- The profile of datasource + profile: postgresql + postgresql: # -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL enabled: true @@ -246,20 +250,25 @@ conf: # -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path: /dolphinscheduler - # -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required - resource.aws.access.key.id: minioadmin + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + aws.credentials.provider.type: AWSStaticCredentialsProvider - # -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required - resource.aws.secret.access.key: minioadmin + # -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required + aws.s3.access.key.id: minioadmin - # -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required - resource.aws.region: ca-central-1 + # -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required + aws.s3.access.key.secret: minioadmin + + # -- The AWS Region to use. if resource.storage.type=S3, This configuration is required + aws.s3.region: ca-central-1 # -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. - resource.aws.s3.bucket.name: dolphinscheduler + aws.s3.bucket.name: dolphinscheduler # -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn - resource.aws.s3.endpoint: http://minio:9000 + aws.s3.endpoint: http://minio:9000 # -- alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id: @@ -645,8 +654,6 @@ worker: WORKER_HOST_WEIGHT: "100" # -- tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: true - # -- Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. - WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT: false # -- If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`; DEFAULT_TENANT_ENABLED: false diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index fe0b7851ba..16fc7899d9 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -165,7 +165,7 @@ The default configuration is as follows: Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`. -### Zookeeper related configuration +### Registry Related configuration DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions. Configuration file location: @@ -191,16 +191,21 @@ The default configuration is as follows: Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`. +For ETCD Registry, please see more details +on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md). +For JDBC Registry, please see more details +on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md). + ### common.properties [hadoop、s3、yarn config properties] Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location: -| Service | Configuration file | -|---------------|----------------------------------------| -| Master Server | `master-server/conf/common.properties` | -| Api Server | `api-server/conf/common.properties` | -| Worker Server | `worker-server/conf/common.properties` | -| Alert Server | `alert-server/conf/common.properties` | +| Service | Configuration file | +|---------------|-----------------------------------------------------------------------| +| Master Server | `master-server/conf/common.properties` | +| Api Server | `api-server/conf/common.properties`, `api-server/conf/aws.yaml` | +| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` | +| Alert Server | `alert-server/conf/common.properties` | The default configuration is as follows: @@ -209,10 +214,6 @@ The default configuration is as follows: | data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files | | resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE | | resource.upload.path | /dolphinscheduler | storage path of resource files | -| aws.access.key.id | minioadmin | access key id of S3 | -| aws.secret.access.key | minioadmin | secret access key of S3 | -| aws.region | us-east-1 | region of S3 | -| aws.s3.endpoint | http://minio:9000 | endpoint of S3 | | hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS | | fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory | | hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission | @@ -327,7 +328,6 @@ Location: `worker-server/conf/application.yaml` | worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely | | worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution | | worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. | -| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants | | worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. | ### Alert Server related configuration diff --git a/docs/docs/en/contribute/development-environment-setup.md b/docs/docs/en/contribute/development-environment-setup.md index 4755f55857..53e92ba7b7 100644 --- a/docs/docs/en/contribute/development-environment-setup.md +++ b/docs/docs/en/contribute/development-environment-setup.md @@ -153,7 +153,7 @@ The browser access address [http://localhost:5173](http://localhost:5173) can lo #### zookeeper -Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it. +Download [ZooKeeper](https://zookeeper.apache.org/releases.html), and extract it. - Create directory `zkData` and `zkLog` - Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index 14ae58a479..0014a9c7f1 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -14,7 +14,7 @@ Configure all the configurations refer to [pseudo-cluster deployment](pseudo-clu ### Modify Configuration -This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster .md), the following describes the parameters that must be modified: +This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster.md), the following describes the parameters that must be modified: ```shell # --------------------------------------------------------- diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index 7a3b43b00e..64f28022e4 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -2,7 +2,7 @@ The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine. -If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). +If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). ## Preparation diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index bd2d73f1ce..ac78e7817f 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -36,7 +36,7 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch ### Start or Stop Server -The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands: +The script `./bin/dolphinscheduler-daemon.sh` can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands: ```shell # Start Standalone Server diff --git a/docs/docs/en/guide/monitor.md b/docs/docs/en/guide/monitor.md index eb8600d8b7..95a6c9c75b 100644 --- a/docs/docs/en/guide/monitor.md +++ b/docs/docs/en/guide/monitor.md @@ -16,6 +16,12 @@ ![worker](../../../img/new_ui/dev/monitor/worker.png) +### Alert Server + +- Mainly related to alert server information. + +![alert-server](../../../img/new_ui/dev/monitor/alert-server.png) + ### Database - Mainly the health status of the DB. @@ -26,18 +32,17 @@ ### Statistics -![statistics](../../../img/new_ui/dev/monitor/statistics.png) +![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png) -| **Parameter** | **Description** | -|----------------------------------------|----------------------------------------------------| -| Number of commands wait to be executed | Statistics of the `t_ds_command` table data. | -| The number of failed commands | Statistics of the `t_ds_error_command` table data. | -| Number of tasks wait to run | Count the data of `task_queue` in the ZooKeeper. | -| Number of tasks wait to be killed | Count the data of `task_kill` in the ZooKeeper. | +Shows the command list in the system. Data is from the `t_ds_command` table. + +![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png) + +Shows the failure command list in the system. Data is from the `t_ds_error_command` table. ### Audit Log The audit log provides information about who accesses the system and the operations made to the system and record related time, which strengthen the security of the system and maintenance. -![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg) +![audit-log](../../../img/new_ui/dev/monitor/audit-log.png) diff --git a/docs/docs/en/guide/parameter/global.md b/docs/docs/en/guide/parameter/global.md index 86cfd69929..46e9fb151d 100644 --- a/docs/docs/en/guide/parameter/global.md +++ b/docs/docs/en/guide/parameter/global.md @@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case, ### Save the workflow and set global parameters -Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it. +Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it. ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png) diff --git a/docs/docs/en/guide/parameter/project-parameter.md b/docs/docs/en/guide/parameter/project-parameter.md index 235cd23512..f9509114d7 100644 --- a/docs/docs/en/guide/parameter/project-parameter.md +++ b/docs/docs/en/guide/parameter/project-parameter.md @@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project. ### Define project-level parameters -On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below: +On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below: ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png) diff --git a/docs/docs/en/guide/parameter/startup-parameter.md b/docs/docs/en/guide/parameter/startup-parameter.md index bf3acc2034..11cf5a9f63 100644 --- a/docs/docs/en/guide/parameter/startup-parameter.md +++ b/docs/docs/en/guide/parameter/startup-parameter.md @@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config ## Usage -Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters. +Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters. ## Example diff --git a/docs/docs/en/guide/resource/configuration.md b/docs/docs/en/guide/resource/configuration.md index 67c68a22c2..1c5b389302 100644 --- a/docs/docs/en/guide/resource/configuration.md +++ b/docs/docs/en/guide/resource/configuration.md @@ -2,8 +2,8 @@ - You could use `Resource Center` to upload text files, UDFs and other task-related files. - You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html) etc. -- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resouce Center` without the need of an external `HDFS` system or `S3`. -- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resouce Center`. In this way, you could operate remote files as if on your local machines. +- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resource Center` without the need of an external `HDFS` system or `S3`. +- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resource Center`. In this way, you could operate remote files as if on your local machines. ## Use Local File System @@ -28,74 +28,37 @@ The configuration you may need to change: ## connect AWS S3 -if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: +if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following: config the following fields ```properties -...... resource.storage.type=S3 - -...... - -resource.aws.access.key.id=aws_access_key_id -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=aws_secret_access_key -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=us-west-2 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint= - -...... ``` -## Use HDFS or Remote Object Storage +```yaml +aws: + s3: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + bucket.name: + endpoint: -After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. +``` + +## connect OSS S3 + +if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: + +config the following fields ```properties -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# user data local directory path, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler - -# resource view suffixs -#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js - -# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS -resource.storage.type=LOCAL -# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended -resource.storage.upload.base.path=/tmp/dolphinscheduler - -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id=minioadmin -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=minioadmin -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=cn-north-1 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint=http://localhost:9000 - # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -107,89 +70,24 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com -# alibaba cloud access key id, required if you set resource.storage.type=OBS +``` + +## connect OBS S3 + +if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: + +config the following fields + +```properties +# access key id, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.id= -# alibaba cloud access key secret, required if you set resource.storage.type=OBS +# access key secret, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.secret= # oss bucket name, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com -# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path -resource.hdfs.root.user=hdfs -# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -resource.hdfs.fs.defaultFS=hdfs://mycluster:8020 - -# whether to startup kerberos -hadoop.security.authentication.startup.state=false - -# java.security.krb5.conf path -java.security.krb5.conf.path=/opt/krb5.conf - -# login user from keytab username -login.user.keytab.username=hdfs-mycluster@ESZ.COM - -# login user from keytab path -login.user.keytab.path=/opt/hdfs.headless.keytab - -# kerberos expire time, the unit is hour -kerberos.expire.time=2 - - -# resourcemanager port, the default value is 8088 if not specified -resource.manager.httpaddress.port=8088 -# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname -yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s -# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) -yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s - -# datasource encryption enable -datasource.encryption.enable=false - -# datasource encryption salt -datasource.encryption.salt=!@#$%^&* - -# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in -# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server -# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`). -data-quality.jar.dir= - -#data-quality.error.output.path=/tmp/data-quality-error-data - -# Network IP gets priority, default inner outer - -# Whether hive SQL is executed in the same session -support.hive.oneSession=false - -# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions -sudo.enable=true - -# network interface preferred like eth0, default: empty -#dolphin.scheduler.network.interface.preferred= - -# network IP gets priority, default: inner outer -#dolphin.scheduler.network.priority.strategy=default - -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - -# development state -development.state=false - -# rpc port -alert.rpc.port=50052 - -# set path of conda.sh -conda.path=/opt/anaconda3/etc/profile.d/conda.sh - -# Task resource limit state -task.resource.limit.state=false - -# way to collect applicationId: log(original regex match), aop -appId.collect: log ``` > **Note:** diff --git a/docs/docs/en/guide/start/docker.md b/docs/docs/en/guide/start/docker.md index b98d0572ae..f29bcd5c75 100644 --- a/docs/docs/en/guide/start/docker.md +++ b/docs/docs/en/guide/start/docker.md @@ -128,7 +128,7 @@ and use `admin` and `dolphinscheduler123` as default username and password in th ![login](../../../../img/new_ui/dev/quick-start/login.png) > Note: If you start the services by the way [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper), and -> strating with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running. +> starting with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running. ## Change Environment Variable diff --git a/docs/docs/en/guide/task/datafactory.md b/docs/docs/en/guide/task/datafactory.md index 1936db81f1..67d00c1227 100644 --- a/docs/docs/en/guide/task/datafactory.md +++ b/docs/docs/en/guide/task/datafactory.md @@ -19,11 +19,11 @@ DolphinScheduler DataFactory functions: ### Application Permission Setting -First, visit the `Subcription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page. -![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png) +First, visit the `Subscription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page. +![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png) After that, select `Contributor` role which satisfy functions calls in data factory. Then click `Members` page, and click `Select members`. Search application name or application `Object ID` to assign `Contributor` role to application. -![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png) +![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png) ## Configurations diff --git a/docs/docs/en/guide/task/dms.md b/docs/docs/en/guide/task/dms.md index a19ec4ba4b..89cf2ef4db 100644 --- a/docs/docs/en/guide/task/dms.md +++ b/docs/docs/en/guide/task/dms.md @@ -73,14 +73,17 @@ Parameters of restarting the task by interface ## Environment to prepare -Some AWS configuration is required, modify a field in file `common.properties` +Some AWS configuration is required, modify a field in file `aws.yaml` ```yaml -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id= -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key= -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region= +dms: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + endpoint: ``` diff --git a/docs/docs/en/guide/task/kubernetes.md b/docs/docs/en/guide/task/kubernetes.md index 332669a60c..871a020294 100644 --- a/docs/docs/en/guide/task/kubernetes.md +++ b/docs/docs/en/guide/task/kubernetes.md @@ -26,7 +26,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the | Command | The container execution command (yaml-style array), for example: ["printenv"] | | Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] | | Custom label | The customized labels for k8s Job. | -| Node selector | The label selectors for running k8s pod. Different value in value set should be seperated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. | +| Node selector | The label selectors for running k8s pod. Different value in value set should be separated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. | | Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. | ## Task Example diff --git a/docs/docs/en/guide/task/mlflow.md b/docs/docs/en/guide/task/mlflow.md index a14ac12483..d500e4cbf6 100644 --- a/docs/docs/en/guide/task/mlflow.md +++ b/docs/docs/en/guide/task/mlflow.md @@ -148,7 +148,7 @@ After this, you can visit the MLflow service (`http://localhost:5000`) page to v ### Preset Algorithm Repository Configuration -If you can't access github, you can modify the following fields in the `commom.properties` configuration file to replace the github address with an accessible address. +If you can't access github, you can modify the following fields in the `common.properties` configuration file to replace the github address with an accessible address. ```yaml # mlflow task plugin preset repository diff --git a/docs/docs/en/guide/task/sagemaker.md b/docs/docs/en/guide/task/sagemaker.md index 7782377130..b520dd33cc 100644 --- a/docs/docs/en/guide/task/sagemaker.md +++ b/docs/docs/en/guide/task/sagemaker.md @@ -35,14 +35,17 @@ The task plugin are shown as follows: ## Environment to prepare -Some AWS configuration is required, modify a field in file `common.properties` +Some AWS configuration is required, modify a field in file `aws.yaml` ```yaml -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id= -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key= -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region= +sagemaker: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + endpoint: ``` diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index d8d1d42d1e..d377e2ed97 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -165,9 +165,9 @@ export DOLPHINSCHEDULER_OPTS=" DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名, 设置值即可。 -## Zookeeper相关配置 +## 注册中心相关配置 -DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置： +DolphinScheduler默认使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置： |服务名称| 配置文件 | |--|--| |Master Server | `master-server/conf/application.yaml`| @@ -190,6 +190,9 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功 DolphinScheduler同样可以通过`bin/env/dolphinscheduler_env.sh`进行Zookeeper相关的配置。 +如果使用etcd作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md)。 +如果使用jdbc作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md)。 + ## common.properties [hadoop、s3、yarn配置] common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId收集相关的配置，配置文件位置： @@ -328,7 +331,6 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 | | worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 | | worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 | -| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 | | worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 | ## Alert Server相关配置 diff --git a/docs/docs/zh/contribute/development-environment-setup.md b/docs/docs/zh/contribute/development-environment-setup.md index 3c97087b18..8e7fe502c3 100644 --- a/docs/docs/zh/contribute/development-environment-setup.md +++ b/docs/docs/zh/contribute/development-environment-setup.md @@ -148,7 +148,7 @@ pnpm run dev #### zookeeper -下载 [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3)，解压 +下载 [ZooKeeper](https://zookeeper.apache.org/releases.html)，解压 * 在 ZooKeeper 的目录下新建 zkData、zkLog文件夹 * 将 conf 目录下的 `zoo_sample.cfg` 文件，复制一份，重命名为 `zoo.cfg`，修改其中数据和日志的配置，如： diff --git a/docs/docs/zh/contribute/join/contribute.md b/docs/docs/zh/contribute/join/contribute.md index fb74d24d6d..69917c76bd 100644 --- a/docs/docs/zh/contribute/join/contribute.md +++ b/docs/docs/zh/contribute/join/contribute.md @@ -32,7 +32,7 @@ 如果你想实现某个 Feature 或者修复某个 Bug。请参考以下内容： * 所有的 Bug 与新 Feature 建议使用 Issues Page 进行管理。 -* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加的回复内容中。 +* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加到回复内容中。 * 最好在核心贡献者找到一个导师（指导者），导师会在设计与功能实现上给予即时的反馈。 * 你应该新建一个分支来开始你的工作，分支的名字参考[参与贡献 Pull Request 需知](./pull-request.md)。比如，你想完成 feature 功能并提交了 Issue 111，那么你的 branch 名字应为 feature-111。 功能名称可与导师讨论后确定。 * 完成后，发送一个 Pull Request 到 dolphinscheduler，提交过程具体请参考下面《[提交代码流程](./submit-code.md)》。 diff --git a/docs/docs/zh/guide/monitor.md b/docs/docs/zh/guide/monitor.md index b45d7a59b3..8033fbb966 100644 --- a/docs/docs/zh/guide/monitor.md +++ b/docs/docs/zh/guide/monitor.md @@ -16,6 +16,12 @@ ![worker](../../../img/new_ui/dev/monitor/worker.png) +### Alert Server + +- 主要是 alert server 的相关信息。 + +![alert-server](../../../img/new_ui/dev/monitor/alert-server.png) + ### Database - 主要是 DB 的健康状况 @@ -26,15 +32,16 @@ ### Statistics -![statistics](../../../img/new_ui/dev/monitor/statistics.png) +![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png) -- 待执行命令数：统计 t_ds_command 表的数据 -- 执行失败的命令数：统计 t_ds_error_command 表的数据 -- 待运行任务数：统计 Zookeeper 中 task_queue 的数据 -- 待杀死任务数：统计 Zookeeper 中 task_kill 的数据 +展示系统中的命令列表，数据来自`t_ds_command`表。 + +![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png) + +展示系统中的失败命令列表，数据来自`t_ds_error_command`表。 ### 审计日志 -审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，他对于维护安全都很有用。 +审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，对于维护安全都很有用。 -![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg) +![audit-log](../../../img/new_ui/dev/monitor/audit-log.png) diff --git a/docs/docs/zh/guide/parameter/global.md b/docs/docs/zh/guide/parameter/global.md index cd26a291b0..3b8b307f91 100644 --- a/docs/docs/zh/guide/parameter/global.md +++ b/docs/docs/zh/guide/parameter/global.md @@ -20,7 +20,7 @@ ### 保存工作流，并设置全局参数 -全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示： +全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示： ![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png) diff --git a/docs/docs/zh/guide/parameter/project-parameter.md b/docs/docs/zh/guide/parameter/project-parameter.md index df26e7a7df..8692b198a8 100644 --- a/docs/docs/zh/guide/parameter/project-parameter.md +++ b/docs/docs/zh/guide/parameter/project-parameter.md @@ -8,7 +8,7 @@ ### 定义项目级别参数 -在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示： +在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示： ![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png) diff --git a/docs/docs/zh/guide/parameter/startup-parameter.md b/docs/docs/zh/guide/parameter/startup-parameter.md index 7da65657a7..38e15787d0 100644 --- a/docs/docs/zh/guide/parameter/startup-parameter.md +++ b/docs/docs/zh/guide/parameter/startup-parameter.md @@ -6,7 +6,7 @@ ## 使用方式 -启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。 +启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。 ## 任务样例 diff --git a/docs/docs/zh/guide/resource/configuration.md b/docs/docs/zh/guide/resource/configuration.md index 739d6fb30c..735c75da97 100644 --- a/docs/docs/zh/guide/resource/configuration.md +++ b/docs/docs/zh/guide/resource/configuration.md @@ -26,77 +26,35 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用 ## 对接AWS S3 -如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： +如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下： 配置以下字段 ```properties -...... resource.storage.type=S3 - -...... - -resource.aws.access.key.id=aws_access_key_id -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=aws_secret_access_key -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=us-west-2 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint= - -...... ``` -## 对接分布式或远端对象存储 +```yaml +aws: + s3: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + bucket.name: + endpoint: -当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置： +``` -### 配置 common.properties 文件 +## 对接阿里云 OSS -在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： +如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： ```properties -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# user data local directory path, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler - -# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS -resource.storage.type=LOCAL - -# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration, -# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended -resource.storage.upload.base.path=/tmp/dolphinscheduler - -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id=minioadmin -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=minioadmin -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=cn-north-1 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint=http://localhost:9000 - # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -108,92 +66,27 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com -# alibaba cloud access key id, required if you set resource.storage.type=OBS +``` + +## 对接华为云 OBS + +如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： + +```properties +# access key id, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.id= -# alibaba cloud access key secret, required if you set resource.storage.type=OBS +# access key secret, required if you set resource.storage.type=OBS resource.huawei.cloud.access.key.secret= # oss bucket name, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.bucket.name=dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OBS resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com -# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path -resource.hdfs.root.user=root -# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; -# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -resource.hdfs.fs.defaultFS=hdfs://localhost:8020 - -# whether to startup kerberos -hadoop.security.authentication.startup.state=false - -# java.security.krb5.conf path -java.security.krb5.conf.path=/opt/krb5.conf - -# login user from keytab username -login.user.keytab.username=hdfs-mycluster@ESZ.COM - -# login user from keytab path -login.user.keytab.path=/opt/hdfs.headless.keytab - -# kerberos expire time, the unit is hour -kerberos.expire.time=2 -# resource view suffixs -#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js - -# resourcemanager port, the default value is 8088 if not specified -resource.manager.httpaddress.port=8088 -# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; -# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname -yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s -# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) -yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s - -# datasource encryption enable -datasource.encryption.enable=false - -# datasource encryption salt -datasource.encryption.salt=!@#$%^&* - -# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in -# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server -# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`). -data-quality.jar.dir= - -#data-quality.error.output.path=/tmp/data-quality-error-data - -# Network IP gets priority, default inner outer - -# Whether hive SQL is executed in the same session -support.hive.oneSession=false - -# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; -# if set false, executing user is the deploy user and doesn't need sudo permissions -sudo.enable=true - -# network interface preferred like eth0, default: empty -#dolphin.scheduler.network.interface.preferred= - -# network IP gets priority, default: inner outer -#dolphin.scheduler.network.priority.strategy=default - -# system env path -#dolphinscheduler.env.path=env/dolphinscheduler_env.sh - -# development state -development.state=false - -# rpc port -alert.rpc.port=50052 - -# way to collect applicationId: log(original regex match), aop -appId.collect: log ``` > **注意**： > > * 如果只配置了 `api-server/conf/common.properties` 的文件，则只是开启了资源上传的操作，并不能满足正常使用。如果想要在工作流中执行相关文件则需要额外配置 `worker-server/conf/common.properties`。 > * 如果用到资源上传的功能，那么[安装部署](../installation/standalone.md)中，部署用户需要有这部分的操作权限。 -> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过次步骤。 +> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过此步骤。 diff --git a/docs/docs/zh/guide/task/appendix.md b/docs/docs/zh/guide/task/appendix.md index 6ba27ad38b..9cc9002dd3 100644 --- a/docs/docs/zh/guide/task/appendix.md +++ b/docs/docs/zh/guide/task/appendix.md @@ -8,7 +8,7 @@ |----------|--------------------------------------------------------------------------------------------------------------------------------------| | 任务名称 | 任务的名称，同一个工作流定义中的节点名称不能重复。 | | 运行标志 | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。 | -| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。 | +| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。 | | 描述 | 当前节点的功能描述。 | | 任务优先级 | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。 | | Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。 | diff --git a/docs/docs/zh/guide/task/datafactory.md b/docs/docs/zh/guide/task/datafactory.md index 0fa3375bc1..0822f60bec 100644 --- a/docs/docs/zh/guide/task/datafactory.md +++ b/docs/docs/zh/guide/task/datafactory.md @@ -19,10 +19,10 @@ DolphinScheduler DataFactory 组件的功能: ### 应用权限设置 -首先打开当前`Subcription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。 -![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png) +首先打开当前`Subscription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。 +![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png) 首先选择`Contributor`角色足够满足调用数据工厂。然后选择`Members`页面，再选择`Select members`，检索APP名称或APP的`Object ID`并添加，从给指定APP添加权限. -![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png) +![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png) ## 环境配置 diff --git a/docs/docs/zh/guide/task/dms.md b/docs/docs/zh/guide/task/dms.md index 6013f45aef..8a87a36f9a 100644 --- a/docs/docs/zh/guide/task/dms.md +++ b/docs/docs/zh/guide/task/dms.md @@ -73,14 +73,17 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS ## 环境配置 -需要进行AWS的一些配置，修改`common.properties`中的以下配置信息 +需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息 ```yaml -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id= -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key= -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region= +dms: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + endpoint: ``` diff --git a/docs/docs/zh/guide/task/mlflow.md b/docs/docs/zh/guide/task/mlflow.md index 3c384ce221..12d2bc5089 100644 --- a/docs/docs/zh/guide/task/mlflow.md +++ b/docs/docs/zh/guide/task/mlflow.md @@ -139,7 +139,7 @@ mlflow server -h 0.0.0.0 -p 5000 --serve-artifacts --backend-store-uri sqlite:// ### 内置算法仓库配置 -如果遇到github无法访问的情况，可以修改`commom.properties`配置文件的以下字段，将github地址替换能访问的地址。 +如果遇到github无法访问的情况，可以修改`common.properties`配置文件的以下字段，将github地址替换能访问的地址。 ```yaml # mlflow task plugin preset repository diff --git a/docs/docs/zh/guide/task/sagemaker.md b/docs/docs/zh/guide/task/sagemaker.md index e4b4c61542..22927171e9 100644 --- a/docs/docs/zh/guide/task/sagemaker.md +++ b/docs/docs/zh/guide/task/sagemaker.md @@ -33,14 +33,17 @@ DolphinScheduler SageMaker 组件的功能: ## 环境配置 -需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息 +需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息 ```yaml -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id= -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key= -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region= +sagemaker: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: + access.key.secret: + region: + endpoint: ``` diff --git a/docs/img/new_ui/dev/monitor/alert-server.png b/docs/img/new_ui/dev/monitor/alert-server.png new file mode 100644 index 0000000000..8ffc7725dc Binary files /dev/null and b/docs/img/new_ui/dev/monitor/alert-server.png differ diff --git a/docs/img/new_ui/dev/monitor/audit-log.jpg b/docs/img/new_ui/dev/monitor/audit-log.jpg deleted file mode 100644 index bc396ad303..0000000000 Binary files a/docs/img/new_ui/dev/monitor/audit-log.jpg and /dev/null differ diff --git a/docs/img/new_ui/dev/monitor/audit-log.png b/docs/img/new_ui/dev/monitor/audit-log.png new file mode 100644 index 0000000000..72acd1f839 Binary files /dev/null and b/docs/img/new_ui/dev/monitor/audit-log.png differ diff --git a/docs/img/new_ui/dev/monitor/command-list.png b/docs/img/new_ui/dev/monitor/command-list.png new file mode 100644 index 0000000000..d8fca25d10 Binary files /dev/null and b/docs/img/new_ui/dev/monitor/command-list.png differ diff --git a/docs/img/new_ui/dev/monitor/failure-command-list.png b/docs/img/new_ui/dev/monitor/failure-command-list.png new file mode 100644 index 0000000000..75c7cda1da Binary files /dev/null and b/docs/img/new_ui/dev/monitor/failure-command-list.png differ diff --git a/docs/img/new_ui/dev/monitor/master.png b/docs/img/new_ui/dev/monitor/master.png index 6505595d6a..bbbb808ca7 100644 Binary files a/docs/img/new_ui/dev/monitor/master.png and b/docs/img/new_ui/dev/monitor/master.png differ diff --git a/docs/img/new_ui/dev/monitor/statistics.png b/docs/img/new_ui/dev/monitor/statistics.png deleted file mode 100644 index a7f4a4a438..0000000000 Binary files a/docs/img/new_ui/dev/monitor/statistics.png and /dev/null differ diff --git a/docs/img/new_ui/dev/monitor/worker.png b/docs/img/new_ui/dev/monitor/worker.png index 4a68279385..166820c930 100644 Binary files a/docs/img/new_ui/dev/monitor/worker.png and b/docs/img/new_ui/dev/monitor/worker.png differ diff --git a/docs/img/new_ui/dev/parameter/context-subprocess02.png b/docs/img/new_ui/dev/parameter/context-subprocess02.png index 8c1acf5f11..7d7cd850ea 100644 Binary files a/docs/img/new_ui/dev/parameter/context-subprocess02.png and b/docs/img/new_ui/dev/parameter/context-subprocess02.png differ diff --git a/docs/img/new_ui/dev/parameter/context_parameter04.png b/docs/img/new_ui/dev/parameter/context_parameter04.png index b1dfd8b3cd..e307e607e7 100644 Binary files a/docs/img/new_ui/dev/parameter/context_parameter04.png and b/docs/img/new_ui/dev/parameter/context_parameter04.png differ diff --git a/docs/img/new_ui/dev/parameter/global_parameter02.png b/docs/img/new_ui/dev/parameter/global_parameter02.png index 1eeaf4b43a..3d6d535311 100644 Binary files a/docs/img/new_ui/dev/parameter/global_parameter02.png and b/docs/img/new_ui/dev/parameter/global_parameter02.png differ diff --git a/docs/img/new_ui/dev/parameter/project_parameter01.png b/docs/img/new_ui/dev/parameter/project_parameter01.png index 48ea149b40..5f6a483d4c 100644 Binary files a/docs/img/new_ui/dev/parameter/project_parameter01.png and b/docs/img/new_ui/dev/parameter/project_parameter01.png differ diff --git a/docs/img/new_ui/dev/parameter/startup_parameter02.png b/docs/img/new_ui/dev/parameter/startup_parameter02.png index cbeb96b658..d1819ad78e 100644 Binary files a/docs/img/new_ui/dev/parameter/startup_parameter02.png and b/docs/img/new_ui/dev/parameter/startup_parameter02.png differ diff --git a/docs/img/new_ui/dev/parameter/startup_parameter04.png b/docs/img/new_ui/dev/parameter/startup_parameter04.png index e32afb4b31..b5799b24a5 100644 Binary files a/docs/img/new_ui/dev/parameter/startup_parameter04.png and b/docs/img/new_ui/dev/parameter/startup_parameter04.png differ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java index eeaaba5d01..4aa29c19c5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java @@ -40,7 +40,7 @@ public final class VoiceAlertChannel implements AlertChannel { Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "aliyun-voice params is null"); + return new AlertResult(false, "aliyun-voice params is null"); } VoiceParam voiceParam = buildVoiceParam(paramsMap); return new VoiceSender(voiceParam).send(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java index c6c29d8735..fe0fc65986 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java @@ -46,7 +46,7 @@ public final class VoiceSender { public AlertResult send() { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); try { Client client = createClient(voiceParam.getConnection()); SingleCallByTtsRequest singleCallByTtsRequest = new SingleCallByTtsRequest() @@ -61,7 +61,7 @@ public final class VoiceSender { } SingleCallByTtsResponseBody body = response.getBody(); if (body.code.equalsIgnoreCase("ok")) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage(body.getCallId()); } else { alertResult.setMessage(body.getMessage()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java index 515a410b63..15f4871392 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java @@ -46,7 +46,7 @@ class VoiceSenderTest { VoiceSender weChatSender = new VoiceSender(voiceParam); AlertResult alertResult = weChatSender.send(); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java index 530a548342..a4eaae232f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java @@ -35,6 +35,6 @@ public interface AlertChannel { AlertResult process(AlertInfo info); default @NonNull AlertResult closeAlert(AlertInfo info) { - return new AlertResult("true", "no need to close alert"); + return new AlertResult(true, "no need to close alert"); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java index 37a3f3357c..004e8b3bda 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java @@ -53,14 +53,6 @@ public class AlertData { */ private String log; - /** - * 0 do not send warning; - * 1 send if process success; - * 2 send if process failed; - * 3 send if process ends, whatever the result; - */ - private int warnType; - /** * AlertType#code */ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java index b6c5db38e9..ceeed97510 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java @@ -33,15 +33,19 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class AlertResult { - /** - * todo: use enum - * false or true - */ - private String status; + private boolean success; /** * alert result message, each plugin can have its own message */ private String message; + public static AlertResult success() { + return new AlertResult(true, null); + } + + public static AlertResult fail(String message) { + return new AlertResult(false, message); + } + } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java index 74c440fe76..f5cc938246 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java @@ -31,7 +31,7 @@ public final class DingTalkAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "ding talk params is null"); + return new AlertResult(false, "ding talk params is null"); } return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index c8ded8cfad..527e38cf77 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -126,7 +126,7 @@ public final class DingTalkSender { private AlertResult checkSendDingTalkSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (null == result) { alertResult.setMessage("send ding talk msg error"); @@ -140,7 +140,7 @@ public final class DingTalkSender { return alertResult; } if (sendMsgResponse.errcode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send ding talk msg success"); return alertResult; } @@ -164,7 +164,7 @@ public final class DingTalkSender { } catch (Exception e) { log.info("send ding talk alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send ding talk alert fail."); } return alertResult; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index cd30105c7a..90f64d7bb2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -52,7 +52,7 @@ public class DingTalkSenderTest { dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkSender = new DingTalkSender(dingTalkConfig); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertEquals(false, alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java index 5728461ae6..06aecd35db 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java @@ -35,24 +35,20 @@ public final class EmailAlertChannel implements AlertChannel { AlertData alert = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "mail params is null"); + return new AlertResult(false, "mail params is null"); } MailSender mailSender = new MailSender(paramsMap); AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent()); - boolean flag; - if (alertResult == null) { alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("alert send error."); log.info("alert send error : {}", alertResult.getMessage()); return alertResult; } - flag = Boolean.parseBoolean(String.valueOf(alertResult.getStatus())); - - if (flag) { + if (alertResult.isSuccess()) { log.info("alert send success"); alertResult.setMessage("email send success."); } else { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index 8826a44fba..58e1eb10b3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -154,7 +154,7 @@ public final class MailSender { */ public AlertResult sendMails(List receivers, List receiverCcs, String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); // if there is no receivers && no receiversCc, no need to process if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiverCcs)) { @@ -201,7 +201,7 @@ public final class MailSender { attachment(title, content, partContent); - alertResult.setStatus("true"); + alertResult.setSuccess(true); return alertResult; } catch (Exception e) { handleException(alertResult, e); @@ -380,7 +380,7 @@ public final class MailSender { email.setDebug(true); email.send(); - alertResult.setStatus("true"); + alertResult.setSuccess(true); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java index 9df19154aa..643cd8a01e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java @@ -66,7 +66,7 @@ public class EmailAlertChannelTest { alertInfo.setAlertParams(paramsMap); AlertResult alertResult = emailAlertChannel.process(alertInfo); Assertions.assertNotNull(alertResult); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } public String getEmailAlertParams() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index acc255ae0e..9a4b5e8257 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -77,7 +77,7 @@ public class MailUtilsTest { AlertResult alertResult = mailSender.sendMails( "Mysql Exception", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -107,7 +107,7 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd"); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails(title, content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } public String list2String() { @@ -142,24 +142,24 @@ public class MailUtilsTest { emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails(title, content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test - public void testAttachmentFile() throws Exception { + public void testAttachmentFile() { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails("gaojing", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test - public void testTableAttachmentFile() throws Exception { + public void testTableAttachmentFile() { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); AlertResult alertResult = mailSender.sendMails("gaojing", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java index 8959c8aaec..29c78a9d1b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java @@ -31,7 +31,7 @@ public final class FeiShuAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "fei shu params is null"); + return new AlertResult(false, "fei shu params is null"); } return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index 369060843c..1c2f3656ea 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -80,7 +80,7 @@ public final class FeiShuSender { public static AlertResult checkSendFeiShuSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (org.apache.commons.lang3.StringUtils.isBlank(result)) { alertResult.setMessage("send fei shu msg error"); @@ -95,7 +95,7 @@ public final class FeiShuSender { return alertResult; } if (sendMsgResponse.statusCode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send fei shu msg success"); return alertResult; } @@ -136,7 +136,7 @@ public final class FeiShuSender { } catch (Exception e) { log.info("send fei shu alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send fei shu alert fail."); } return alertResult; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java index 41f372b85c..829b02dea6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java @@ -43,7 +43,7 @@ public class FeiShuSenderTest { alertData.setContent("feishu test content"); FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig); AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -87,12 +87,12 @@ public class FeiShuSenderTest { FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig); AlertResult alertResult = feiShuSender.checkSendFeiShuSendMsgResult(""); - Assertions.assertFalse(Boolean.valueOf(alertResult.getStatus())); + Assertions.assertFalse(alertResult.isSuccess()); AlertResult alertResult2 = feiShuSender.checkSendFeiShuSendMsgResult("123"); Assertions.assertEquals("send fei shu msg fail", alertResult2.getMessage()); String response = "{\"StatusCode\":\"0\",\"extra\":\"extra\",\"StatusMessage\":\"StatusMessage\"}"; AlertResult alertResult3 = feiShuSender.checkSendFeiShuSendMsgResult(response); - Assertions.assertTrue(Boolean.valueOf(alertResult3.getStatus())); + Assertions.assertTrue(alertResult3.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java index 944762f13f..caf1c4c598 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java @@ -31,7 +31,7 @@ public final class HttpAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map paramsMap = alertInfo.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "http params is null"); + return new AlertResult(false, "http params is null"); } return new HttpSender(paramsMap).send(alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java index a1de852407..e2a6606a39 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java @@ -92,18 +92,18 @@ public final class HttpSender { } if (httpRequest == null) { - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("Request types are not supported"); return alertResult; } try { String resp = this.getResponseString(httpRequest); - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage(resp); } catch (Exception e) { log.error("send http alert msg exception : {}", e.getMessage()); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage( String.format("Send http request alert failed: %s", e.getMessage())); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java index aebf6f9d50..ee67db47f1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java @@ -62,9 +62,9 @@ public class HttpAlertChannelTest { // HttpSender(paramsMap).send(alertData.getContent()); already test in HttpSenderTest.sendTest. so we can mock // it - doReturn(new AlertResult("true", "success")).when(alertChannel).process(any()); + doReturn(new AlertResult(true, "success")).when(alertChannel).process(any()); AlertResult alertResult = alertChannel.process(alertInfo); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); } /** diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java index be013457ac..40f589a10b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java @@ -46,7 +46,7 @@ public class HttpSenderTest { HttpSender httpSender = spy(new HttpSender(paramsMap)); doReturn("success").when(httpSender).getResponseString(any()); AlertResult alertResult = httpSender.send("Fault tolerance warning"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); Assertions.assertTrue(httpSender.getRequestUrl().contains(url)); Assertions.assertTrue(httpSender.getRequestUrl().contains(contentField)); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java index b033139520..430bacbf63 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java @@ -30,8 +30,8 @@ public final class PagerDutyAlertChannel implements AlertChannel { public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); - if (alertParams == null || alertParams.size() == 0) { - return new AlertResult("false", "PagerDuty alert params is empty"); + if (alertParams == null || alertParams.isEmpty()) { + return new AlertResult(false, "PagerDuty alert params is empty"); } return new PagerDutySender(alertParams).sendPagerDutyAlter(alertData.getTitle(), alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java index 65792c8eae..11dd01048a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java @@ -53,7 +53,7 @@ public final class PagerDutySender { public AlertResult sendPagerDutyAlter(String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send pager duty alert fail."); try { @@ -83,7 +83,7 @@ public final class PagerDutySender { String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8); try { if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send pager duty alert success"); } else { alertResult.setMessage( diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java index 16cf16f62f..52a47aa20e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java @@ -39,6 +39,6 @@ public class PagerDutySenderTest { public void testSend() { PagerDutySender pagerDutySender = new PagerDutySender(pagerDutyConfig); AlertResult alertResult = pagerDutySender.sendPagerDutyAlter("pagerduty test title", "pagerduty test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java index 7ca79253fa..5928faacec 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java @@ -31,7 +31,7 @@ public final class PrometheusAlertChannel implements AlertChannel { AlertData alertData = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "prometheus alert manager params is null"); + return new AlertResult(false, "prometheus alert manager params is null"); } return new PrometheusAlertSender(paramsMap).sendMessage(alertData); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java index 1106e6799f..48fda566b1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.alert.api.HttpServiceRetryStrategy; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -64,11 +65,10 @@ public class PrometheusAlertSender { String resp = sendMsg(alertData); return checkSendAlertManageMsgResult(resp); } catch (Exception e) { - String errorMsg = String.format("send prometheus alert manager alert error, exception: %s", e.getMessage()); - log.error(errorMsg); + log.error("Send prometheus alert manager alert error", e); alertResult = new AlertResult(); - alertResult.setStatus("false"); - alertResult.setMessage(errorMsg); + alertResult.setSuccess(false); + alertResult.setMessage(ExceptionUtils.getMessage(e)); } return alertResult; } @@ -106,10 +106,10 @@ public class PrometheusAlertSender { public AlertResult checkSendAlertManageMsgResult(String resp) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (Objects.equals(resp, PrometheusAlertConstants.ALERT_SUCCESS)) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("prometheus alert manager send success"); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java index 2347d97262..c0d18396e4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java @@ -55,17 +55,17 @@ public class PrometheusAlertSenderTest { " }]"); PrometheusAlertSender sender = new PrometheusAlertSender(config); AlertResult result = sender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @Test public void testCheckSendAlertManageMsgResult() { PrometheusAlertSender prometheusAlertSender = new PrometheusAlertSender(config); AlertResult alertResult1 = prometheusAlertSender.checkSendAlertManageMsgResult(""); - Assertions.assertFalse(Boolean.parseBoolean(alertResult1.getStatus())); + Assertions.assertFalse(alertResult1.isSuccess()); Assertions.assertEquals("prometheus alert manager send fail, resp is ", alertResult1.getMessage()); AlertResult alertResult2 = prometheusAlertSender.checkSendAlertManageMsgResult("alert success"); - Assertions.assertTrue(Boolean.parseBoolean(alertResult2.getStatus())); + Assertions.assertTrue(alertResult2.isSuccess()); Assertions.assertEquals("prometheus alert manager send success", alertResult2.getMessage()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java index d091eb9d82..81cd59a5a9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java @@ -33,7 +33,7 @@ public final class ScriptAlertChannel implements AlertChannel { AlertData alertData = alertinfo.getAlertData(); Map paramsMap = alertinfo.getAlertParams(); if (MapUtils.isEmpty(paramsMap)) { - return new AlertResult("false", "script params is empty"); + return new AlertResult(false, "script params is empty"); } return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle(), alertData.getContent()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java index a18adb2c7e..19b7149e74 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java @@ -56,7 +56,7 @@ public final class ScriptSender { } // If it is another type of alarm script can be added here, such as python - alertResult.setStatus("false"); + alertResult.setSuccess(false); log.error("script type error: {}", scriptType); alertResult.setMessage("script type error : " + scriptType); return alertResult; @@ -64,7 +64,7 @@ public final class ScriptSender { private AlertResult executeShellScript(String title, String content) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); if (Boolean.TRUE.equals(OSUtils.isWindows())) { alertResult.setMessage("shell script not support windows os"); return alertResult; @@ -111,7 +111,7 @@ public final class ScriptSender { int exitCode = ProcessUtils.executeScript(cmd); if (exitCode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send script alert msg success"); return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java index 32e996f5e8..c392b6f758 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.plugin.alert.script; +import static org.junit.jupiter.api.Assertions.assertFalse; + import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.HashMap; @@ -48,9 +50,9 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test title Kris", "test content"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); alertResult = scriptSender.sendScriptAlert("error msg title", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -58,7 +60,7 @@ public class ScriptSenderTest { scriptConfig.put(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, "' ; calc.exe ; '"); ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult = scriptSender.sendScriptAlert("test title Kris", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -67,7 +69,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test user params NPE", "test content"); - Assertions.assertEquals("true", alertResult.getStatus()); + Assertions.assertTrue(alertResult.isSuccess()); } @Test @@ -76,7 +78,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test path NPE", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -85,7 +87,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test path NPE", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + assertFalse(alertResult.isSuccess()); Assertions.assertTrue(alertResult.getMessage().contains("shell script is invalid, only support .sh file")); } @@ -95,7 +97,7 @@ public class ScriptSenderTest { ScriptSender scriptSender = new ScriptSender(scriptConfig); AlertResult alertResult; alertResult = scriptSender.sendScriptAlert("test type is error", "test content"); - Assertions.assertEquals("false", alertResult.getStatus()); + assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java index c8cb36a78b..8052c7c4f1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java @@ -30,11 +30,11 @@ public final class SlackAlertChannel implements AlertChannel { public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); - if (alertParams == null || alertParams.size() == 0) { - return new AlertResult("false", "Slack alert params is empty"); + if (alertParams == null || alertParams.isEmpty()) { + return new AlertResult(false, "Slack alert params is empty"); } SlackSender slackSender = new SlackSender(alertParams); String response = slackSender.sendMessage(alertData.getTitle(), alertData.getContent()); - return new AlertResult("ok".equals(response) ? "true" : "false", response); + return new AlertResult("ok".equals(response), response); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java index efc8912d1a..ed33ef5497 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java @@ -30,7 +30,7 @@ public final class TelegramAlertChannel implements AlertChannel { public AlertResult process(AlertInfo info) { Map alertParams = info.getAlertParams(); if (alertParams == null || alertParams.isEmpty()) { - return new AlertResult("false", "Telegram alert params is empty"); + return AlertResult.fail("Telegram alert params is empty"); } AlertData data = info.getAlertData(); return new TelegramSender(alertParams).sendMessage(data); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 129bc62c1c..417e97d4cd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -105,7 +105,7 @@ public final class TelegramSender { } catch (Exception e) { log.warn("send telegram alert msg exception : {}", e.getMessage()); result = new AlertResult(); - result.setStatus("false"); + result.setSuccess(false); result.setMessage(String.format("send telegram alert fail. %s", e.getMessage())); } return result; @@ -113,7 +113,7 @@ public final class TelegramSender { private AlertResult parseRespToResult(String resp) { AlertResult result = new AlertResult(); - result.setStatus("false"); + result.setSuccess(false); if (null == resp || resp.isEmpty()) { result.setMessage("send telegram msg error. telegram server resp is empty"); return result; @@ -127,7 +127,7 @@ public final class TelegramSender { result.setMessage(String.format("send telegram alert fail. telegram server error_code: %d, description: %s", response.errorCode, response.description)); } else { - result.setStatus("true"); + result.setSuccess(true); result.setMessage("send telegram msg success."); } return result; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java index a57de30219..d05a45d73f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSenderTest.java @@ -52,7 +52,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, "XXXXXXX"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -65,7 +65,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, "-XXXXXXX"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @Test @@ -75,7 +75,7 @@ public class TelegramSenderTest { alertData.setContent("telegram test content"); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -89,7 +89,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_MARKDOWN); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } @@ -102,7 +102,7 @@ public class TelegramSenderTest { TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_HTML); TelegramSender telegramSender = new TelegramSender(telegramConfig); AlertResult result = telegramSender.sendMessage(alertData); - Assertions.assertEquals("false", result.getStatus()); + Assertions.assertFalse(result.isSuccess()); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java index 38a582f1c6..94f77aed6e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java @@ -33,7 +33,7 @@ public final class WebexTeamsAlertChannel implements AlertChannel { AlertData alertData = alertInfo.getAlertData(); Map alertParams = alertInfo.getAlertParams(); if (MapUtils.isEmpty(alertParams)) { - return new AlertResult("false", "WebexTeams alert params is empty"); + return new AlertResult(false, "WebexTeams alert params is empty"); } return new WebexTeamsSender(alertParams).sendWebexTeamsAlter(alertData); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java index f8201a40e0..3b8b3d21c8 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java @@ -67,7 +67,7 @@ public final class WebexTeamsSender { public AlertResult sendWebexTeamsAlter(AlertData alertData) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus("false"); + alertResult.setSuccess(false); alertResult.setMessage("send webex teams alert fail."); try { @@ -93,7 +93,7 @@ public final class WebexTeamsSender { String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8); try { if (statusCode == HttpStatus.SC_OK) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("send webex teams alert success"); } else { alertResult.setMessage(String.format( diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java index 1d3070cb55..ddc806e593 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSenderTest.java @@ -85,6 +85,6 @@ public class WebexTeamsSenderTest { public void testSend() { WebexTeamsSender webexTeamsSender = new WebexTeamsSender(webexTeamsConfig); AlertResult alertResult = webexTeamsSender.sendWebexTeamsAlter(alertData); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java index 786cdb159f..dcc53c7f59 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java @@ -31,7 +31,7 @@ public final class WeChatAlertChannel implements AlertChannel { AlertData alertData = info.getAlertData(); Map paramsMap = info.getAlertParams(); if (null == paramsMap) { - return new AlertResult("false", "we chat params is null"); + return new AlertResult(false, "we chat params is null"); } return new WeChatSender(paramsMap).sendEnterpriseWeChat(alertData.getTitle(), alertData.getContent()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index c5ffec1f46..d3fba217dc 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -54,7 +54,6 @@ import lombok.extern.slf4j.Slf4j; public final class WeChatSender { private static final String MUST_NOT_NULL = " must not null"; - private static final String ALERT_STATUS = "false"; private static final String AGENT_ID_REG_EXP = "{agentId}"; private static final String MSG_REG_EXP = "{msg}"; private static final String USER_REG_EXP = "{toUser}"; @@ -178,7 +177,7 @@ public final class WeChatSender { private static AlertResult checkWeChatSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); if (null == result) { alertResult.setMessage("we chat send fail"); @@ -192,11 +191,11 @@ public final class WeChatSender { return alertResult; } if (sendMsgResponse.errcode == 0) { - alertResult.setStatus("true"); + alertResult.setSuccess(true); alertResult.setMessage("we chat alert send success"); return alertResult; } - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); alertResult.setMessage(sendMsgResponse.getErrmsg()); return alertResult; } @@ -212,7 +211,7 @@ public final class WeChatSender { if (null == weChatToken) { alertResult = new AlertResult(); alertResult.setMessage("send we chat alert fail,get weChat token error"); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); return alertResult; } String enterpriseWeChatPushUrlReplace = ""; @@ -239,7 +238,7 @@ public final class WeChatSender { log.info("send we chat alert msg exception : {}", e.getMessage()); alertResult = new AlertResult(); alertResult.setMessage("send we chat alert fail"); - alertResult.setStatus(ALERT_STATUS); + alertResult.setSuccess(false); } return alertResult; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java index e0c934f436..6e4c318d13 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSenderTest.java @@ -71,7 +71,7 @@ public class WeChatSenderTest { WeChatSender weChatSender = new WeChatSender(weChatConfig); AlertResult alertResult = weChatSender.sendEnterpriseWeChat("test", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } @Test @@ -79,7 +79,7 @@ public class WeChatSenderTest { weChatConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp()); WeChatSender weChatSender = new WeChatSender(weChatConfig); AlertResult alertResult = weChatSender.sendEnterpriseWeChat("test", content); - Assertions.assertEquals("false", alertResult.getStatus()); + Assertions.assertFalse(alertResult.isSuccess()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 507d7acb45..844a5983df 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -66,6 +66,12 @@ org.springframework.cloud spring-cloud-starter-kubernetes-client-config + + + org.springframework.boot + spring-boot-starter-test + test + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index bf28193b3f..24c8fb2f11 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -56,6 +56,13 @@ conf + + ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources + + **/*.yaml + + conf + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh index c668944139..d953e04d2f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh @@ -24,6 +24,7 @@ -XX:+PrintGCDetails -Xloggc:gc.log +-XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 55c5c3446c..2435711047 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -18,11 +18,7 @@ package org.apache.dolphinscheduler.alert; import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; -import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; -import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; -import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -50,14 +46,6 @@ public class AlertServer { @Autowired private AlertBootstrapService alertBootstrapService; - @Autowired - private ListenerEventPostService listenerEventPostService; - @Autowired - private AlertRpcServer alertRpcServer; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertRegistryClient alertRegistryClient; public static void main(String[] args) { AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); @@ -68,27 +56,14 @@ public class AlertServer { @PostConstruct public void run() { - log.info("Alert server is staring ..."); - alertPluginManager.start(); - alertRegistryClient.start(); + log.info("AlertServer is staring ..."); alertBootstrapService.start(); - listenerEventPostService.start(); - alertRpcServer.start(); - log.info("Alert server is started ..."); + log.info("AlertServer is started ..."); } @PreDestroy public void close() { - destroy("alert server destroy"); - } - - /** - * gracefully stop - * - * @param cause stop cause - */ - public void destroy(String cause) { - + String cause = "AlertServer destroy"; try { // set stop signal is true // execute only once @@ -96,19 +71,14 @@ public class AlertServer { log.warn("AlterServer is already stopped"); return; } - log.info("Alert server is stopping, cause: {}", cause); - try ( - AlertRpcServer closedAlertRpcServer = alertRpcServer; - AlertBootstrapService closedAlertBootstrapService = alertBootstrapService; - ListenerEventPostService closedListenerEventPostService = listenerEventPostService; - AlertRegistryClient closedAlertRegistryClient = alertRegistryClient) { - // close resource - } + log.info("AlertServer is stopping, cause: {}", cause); + alertBootstrapService.close(); // thread sleep 3 seconds for thread quietly stop ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis()); - log.info("Alter server stopped, cause: {}", cause); + log.info("AlertServer stopped, cause: {}", cause); } catch (Exception e) { - log.error("Alert server stop failed, cause: {}", cause, e); + log.error("AlertServer stop failed, cause: {}", cause, e); } } + } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 824851fd92..240f92b846 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -43,6 +43,8 @@ public final class AlertConfig implements Validator { private Duration maxHeartbeatInterval = Duration.ofSeconds(60); + private int senderParallelism = 100; + private String alertServerAddress; @Override @@ -58,6 +60,10 @@ public final class AlertConfig implements Validator { errors.rejectValue("max-heartbeat-interval", null, "should be a valid duration"); } + if (senderParallelism <= 0) { + errors.rejectValue("sender-parallelism", null, "should be a positive number"); + } + if (StringUtils.isEmpty(alertServerAddress)) { alertConfig.setAlertServerAddress(NetUtils.getAddr(alertConfig.getPort())); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java index db75a49371..606834a2e9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java @@ -45,6 +45,12 @@ public class AlertServerMetrics { .register(Metrics.globalRegistry); } + public void registerSendingAlertGauge(final Supplier supplier) { + Gauge.builder("ds.alert.sending", supplier) + .description("Number of sending alert") + .register(Metrics.globalRegistry); + } + public static void registerUncachedException(final Supplier supplier) { Gauge.builder("ds.alert.uncached.exception", supplier) .description("number of uncached exception") diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java index 1035018e9c..badd463166 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java @@ -36,8 +36,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -@Component @Slf4j +@Component public final class AlertPluginManager { private final PluginDao pluginDao; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java index 0bfefed223..a5481fdd49 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.alert.registry; import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat; import org.apache.dolphinscheduler.common.model.BaseHeartBeatTask; @@ -42,12 +43,15 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask private final RegistryClient registryClient; private final MetricsProvider metricsProvider; + + private final AlertHAServer alertHAServer; private final String heartBeatPath; private final long startupTime; public AlertHeartbeatTask(AlertConfig alertConfig, MetricsProvider metricsProvider, - RegistryClient registryClient) { + RegistryClient registryClient, + AlertHAServer alertHAServer) { super("AlertHeartbeatTask", alertConfig.getMaxHeartbeatInterval().toMillis()); this.startupTime = System.currentTimeMillis(); this.alertConfig = alertConfig; @@ -55,6 +59,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask this.registryClient = registryClient; this.heartBeatPath = RegistryNodeType.ALERT_SERVER.getRegistryPath() + "/" + alertConfig.getAlertServerAddress(); + this.alertHAServer = alertHAServer; this.processId = OSUtils.getProcessID(); } @@ -70,6 +75,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) .serverStatus(ServerStatus.NORMAL) + .isActive(alertHAServer.isActive()) .host(NetUtils.getHost()) .port(alertConfig.getPort()) .build(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java index 616220bd1b..1b7839d816 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.alert.registry; import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.registry.api.RegistryClient; -import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import lombok.extern.slf4j.Slf4j; @@ -42,10 +42,12 @@ public class AlertRegistryClient implements AutoCloseable { private AlertHeartbeatTask alertHeartbeatTask; + @Autowired + private AlertHAServer alertHAServer; + public void start() { log.info("AlertRegistryClient starting..."); - registryClient.getLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); - alertHeartbeatTask = new AlertHeartbeatTask(alertConfig, metricsProvider, registryClient); + alertHeartbeatTask = new AlertHeartbeatTask(alertConfig, metricsProvider, registryClient, alertHAServer); alertHeartbeatTask.start(); // start heartbeat task log.info("AlertRegistryClient started..."); @@ -55,7 +57,6 @@ public class AlertRegistryClient implements AutoCloseable { public void close() { log.info("AlertRegistryClient closing..."); alertHeartbeatTask.shutdown(); - registryClient.releaseLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); log.info("AlertRegistryClient closed..."); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java index 9f11fa6c2e..6a5ed3e0be 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.alert.rpc; -import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; +import org.apache.dolphinscheduler.alert.service.AlertSender; import org.apache.dolphinscheduler.extract.alert.IAlertOperator; import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest; import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; @@ -32,16 +32,15 @@ import org.springframework.stereotype.Service; public class AlertOperatorImpl implements IAlertOperator { @Autowired - private AlertBootstrapService alertBootstrapService; + private AlertSender alertSender; @Override public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) { log.info("Received AlertSendRequest : {}", alertSendRequest); - AlertSendResponse alertSendResponse = alertBootstrapService.syncHandler( + AlertSendResponse alertSendResponse = alertSender.syncHandler( alertSendRequest.getGroupId(), alertSendRequest.getTitle(), - alertSendRequest.getContent(), - alertSendRequest.getWarnType()); + alertSendRequest.getContent()); log.info("Handle AlertSendRequest finish: {}", alertSendResponse); return alertSendResponse; } @@ -49,7 +48,7 @@ public class AlertOperatorImpl implements IAlertOperator { @Override public AlertSendResponse sendTestAlert(AlertTestSendRequest alertSendRequest) { log.info("Received AlertTestSendRequest : {}", alertSendRequest); - AlertSendResponse alertSendResponse = alertBootstrapService.syncTestSend( + AlertSendResponse alertSendResponse = alertSender.syncTestSend( alertSendRequest.getPluginDefineId(), alertSendRequest.getPluginInstanceParams()); log.info("Handle AlertTestSendRequest finish: {}", alertSendResponse); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java index 3bd368573a..d73e4755dd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.alert.rpc; import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; @@ -31,20 +30,7 @@ import org.springframework.stereotype.Service; public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { public AlertRpcServer(AlertConfig alertConfig) { - super(NettyRemotingServerFactory.buildNettyRemotingServer( - NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build())); + super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build()); } - public void start() { - log.info("Starting AlertRpcServer..."); - nettyRemotingServer.start(); - log.info("Started AlertRpcServer..."); - } - - @Override - public void close() { - log.info("Closing AlertRpcServer..."); - nettyRemotingServer.close(); - log.info("Closed AlertRpcServer..."); - } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java new file mode 100644 index 0000000000..1c61659ce3 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventFetcher.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractEventFetcher extends BaseDaemonThread implements EventFetcher { + + protected static final int FETCH_SIZE = 100; + + protected static final long FETCH_INTERVAL = 5_000; + + protected final AlertHAServer alertHAServer; + + private final EventPendingQueue eventPendingQueue; + + private final AtomicBoolean runningFlag = new AtomicBoolean(false); + + private Integer eventOffset; + + protected AbstractEventFetcher(String fetcherName, + AlertHAServer alertHAServer, + EventPendingQueue eventPendingQueue) { + super(fetcherName); + this.alertHAServer = alertHAServer; + this.eventPendingQueue = eventPendingQueue; + this.eventOffset = -1; + } + + @Override + public synchronized void start() { + if (!runningFlag.compareAndSet(false, true)) { + throw new IllegalArgumentException("AlertEventFetcher is already started"); + } + log.info("AlertEventFetcher starting..."); + super.start(); + log.info("AlertEventFetcher started..."); + } + + @Override + public void run() { + while (runningFlag.get()) { + try { + if (!alertHAServer.isActive()) { + log.debug("The current node is not active, will not loop Alert"); + Thread.sleep(FETCH_INTERVAL); + continue; + } + List pendingEvents = fetchPendingEvent(eventOffset); + if (CollectionUtils.isEmpty(pendingEvents)) { + log.debug("No pending events found"); + Thread.sleep(FETCH_INTERVAL); + continue; + } + for (T alert : pendingEvents) { + eventPendingQueue.put(alert); + } + eventOffset = Math.max(eventOffset, + pendingEvents.stream().map(this::getEventOffset).max(Integer::compareTo).get()); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + } catch (Exception ex) { + log.error("AlertEventFetcher error", ex); + } + } + } + + protected abstract int getEventOffset(T event); + + @Override + public void shutdown() { + if (!runningFlag.compareAndSet(true, false)) { + log.warn("The AlertEventFetcher is not started"); + } + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java new file mode 100644 index 0000000000..568125002e --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventLoop.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractEventLoop extends BaseDaemonThread implements EventLoop { + + private final EventPendingQueue eventPendingQueue; + + private final AtomicInteger handlingEventCount; + + private final int eventHandleWorkerNum; + + private final ThreadPoolExecutor threadPoolExecutor; + + private final AtomicBoolean runningFlag = new AtomicBoolean(false); + + protected AbstractEventLoop(String name, + ThreadPoolExecutor threadPoolExecutor, + EventPendingQueue eventPendingQueue) { + super(name); + this.handlingEventCount = new AtomicInteger(0); + this.eventHandleWorkerNum = threadPoolExecutor.getMaximumPoolSize(); + this.threadPoolExecutor = threadPoolExecutor; + this.eventPendingQueue = eventPendingQueue; + } + + @Override + public synchronized void start() { + if (!runningFlag.compareAndSet(false, true)) { + throw new IllegalArgumentException(getClass().getName() + " is already started"); + } + log.info("{} starting...", getClass().getName()); + super.start(); + log.info("{} started...", getClass().getName()); + } + + @Override + public void run() { + while (runningFlag.get()) { + try { + if (handlingEventCount.get() >= eventHandleWorkerNum) { + log.debug("There is no idle event worker, waiting for a while..."); + Thread.sleep(1000); + continue; + } + T pendingEvent = eventPendingQueue.take(); + handlingEventCount.incrementAndGet(); + CompletableFuture.runAsync(() -> handleEvent(pendingEvent), threadPoolExecutor) + .whenComplete((aVoid, throwable) -> { + if (throwable != null) { + log.error("Handle event: {} error", pendingEvent, throwable); + } + handlingEventCount.decrementAndGet(); + }); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + log.error("Loop event thread has been interrupted..."); + break; + } catch (Exception ex) { + log.error("Loop event error", ex); + } + } + } + + @Override + public int getHandlingEventCount() { + return handlingEventCount.get(); + } + + @Override + public void shutdown() { + if (!runningFlag.compareAndSet(true, false)) { + log.warn(getClass().getName() + " is not started"); + } + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java new file mode 100644 index 0000000000..1d7e213ab9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventPendingQueue.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import java.util.concurrent.LinkedBlockingQueue; + +public abstract class AbstractEventPendingQueue implements EventPendingQueue { + + private final LinkedBlockingQueue pendingAlertQueue; + + private final int capacity; + + protected AbstractEventPendingQueue(int capacity) { + this.capacity = capacity; + this.pendingAlertQueue = new LinkedBlockingQueue<>(capacity); + } + + @Override + public void put(T alert) throws InterruptedException { + pendingAlertQueue.put(alert); + } + + @Override + public T take() throws InterruptedException { + return pendingAlertQueue.take(); + } + + @Override + public int size() { + return pendingAlertQueue.size(); + } + + @Override + public int capacity() { + return capacity; + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java new file mode 100644 index 0000000000..deff97da49 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AbstractEventSender.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.alert.api.AlertChannel; +import org.apache.dolphinscheduler.alert.api.AlertConstants; +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertInfo; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; +import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +@Slf4j +public abstract class AbstractEventSender implements EventSender { + + protected final AlertPluginManager alertPluginManager; + + private final long sendEventTimeout; + + protected AbstractEventSender(AlertPluginManager alertPluginManager, long sendEventTimeout) { + this.alertPluginManager = alertPluginManager; + this.sendEventTimeout = sendEventTimeout; + } + + @Override + public void sendEvent(T event) { + List alertPluginInstanceList = getAlertPluginInstanceList(event); + if (CollectionUtils.isEmpty(alertPluginInstanceList)) { + onError(event, "No bind plugin instance found"); + return; + } + AlertData alertData = getAlertData(event); + List alertSendStatuses = new ArrayList<>(); + for (AlertPluginInstance instance : alertPluginInstanceList) { + AlertResult alertResult = doSendEvent(instance, alertData); + AlertStatus alertStatus = + alertResult.isSuccess() ? AlertStatus.EXECUTION_SUCCESS : AlertStatus.EXECUTION_FAILURE; + AlertSendStatus alertSendStatus = AlertSendStatus.builder() + .alertId(getEventId(event)) + .alertPluginInstanceId(instance.getId()) + .sendStatus(alertStatus) + .log(JSONUtils.toJsonString(alertResult)) + .createTime(new Date()) + .build(); + alertSendStatuses.add(alertSendStatus); + } + long failureCount = alertSendStatuses.stream() + .map(alertSendStatus -> alertSendStatus.getSendStatus() == AlertStatus.EXECUTION_FAILURE) + .count(); + long successCount = alertSendStatuses.stream() + .map(alertSendStatus -> alertSendStatus.getSendStatus() == AlertStatus.EXECUTION_SUCCESS) + .count(); + if (successCount == 0) { + onError(event, JSONUtils.toJsonString(alertSendStatuses)); + } else { + if (failureCount > 0) { + onPartialSuccess(event, JSONUtils.toJsonString(alertSendStatuses)); + } else { + onSuccess(event, JSONUtils.toJsonString(alertSendStatuses)); + } + } + } + + public abstract List getAlertPluginInstanceList(T event); + + public abstract AlertData getAlertData(T event); + + public abstract Integer getEventId(T event); + + public abstract void onError(T event, String log); + + public abstract void onPartialSuccess(T event, String log); + + public abstract void onSuccess(T event, String log); + + @Override + public AlertResult doSendEvent(AlertPluginInstance instance, AlertData alertData) { + int pluginDefineId = instance.getPluginDefineId(); + Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); + if (!alertChannelOptional.isPresent()) { + return AlertResult.fail("Cannot find the alertPlugin: " + pluginDefineId); + } + AlertChannel alertChannel = alertChannelOptional.get(); + + AlertInfo alertInfo = AlertInfo.builder() + .alertData(alertData) + .alertParams(PluginParamsTransfer.getPluginParamsMap(instance.getPluginInstanceParams())) + .alertPluginInstanceId(instance.getId()) + .build(); + try { + AlertResult alertResult; + if (sendEventTimeout <= 0) { + if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { + alertResult = alertChannel.closeAlert(alertInfo); + } else { + alertResult = alertChannel.process(alertInfo); + } + } else { + CompletableFuture future; + if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { + future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); + } else { + future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); + } + alertResult = future.get(sendEventTimeout, TimeUnit.MILLISECONDS); + } + checkNotNull(alertResult, "AlertResult cannot be null"); + return alertResult; + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + return AlertResult.fail(ExceptionUtils.getMessage(interruptedException)); + } catch (Exception e) { + log.error("Send alert data {} failed", alertData, e); + return AlertResult.fail(ExceptionUtils.getMessage(e)); + } + } + + @Override + public AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams) { + + Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); + if (!alertChannelOptional.isPresent()) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + AlertSendResponse.AlertSendResponseResult.fail("Cannot find the alertPlugin: " + pluginDefineId); + return AlertSendResponse.fail(Lists.newArrayList(alertSendResponseResult)); + } + AlertData alertData = AlertData.builder() + .title(AlertConstants.TEST_TITLE) + .content(AlertConstants.TEST_CONTENT) + .build(); + + AlertInfo alertInfo = AlertInfo.builder() + .alertData(alertData) + .alertParams(PluginParamsTransfer.getPluginParamsMap(pluginInstanceParams)) + .build(); + + try { + AlertResult alertResult = alertChannelOptional.get().process(alertInfo); + Preconditions.checkNotNull(alertResult, "AlertResult cannot be null"); + if (alertResult.isSuccess()) { + return AlertSendResponse + .success(Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.success())); + } + return AlertSendResponse.fail( + Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.fail(alertResult.getMessage()))); + } catch (Exception e) { + log.error("Test send alert error", e); + return new AlertSendResponse(false, + Lists.newArrayList(AlertSendResponse.AlertSendResponseResult.fail(ExceptionUtils.getMessage(e)))); + } + + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java index 77e62a65a0..5553e01bc9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java @@ -17,350 +17,84 @@ package org.apache.dolphinscheduler.alert.service; -import org.apache.dolphinscheduler.alert.api.AlertChannel; -import org.apache.dolphinscheduler.alert.api.AlertConstants; -import org.apache.dolphinscheduler.alert.api.AlertData; -import org.apache.dolphinscheduler.alert.api.AlertInfo; -import org.apache.dolphinscheduler.alert.api.AlertResult; -import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.entity.Alert; -import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; -import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; -import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nullable; +import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; +import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import com.google.common.collect.Lists; - -@Service +/** + * The bootstrap service for alert server. it will start all the necessary component for alert server. + */ @Slf4j -public final class AlertBootstrapService extends BaseDaemonThread implements AutoCloseable { +@Service +public final class AlertBootstrapService implements AutoCloseable { - @Autowired - private AlertDao alertDao; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertConfig alertConfig; + private final AlertRpcServer alertRpcServer; - public AlertBootstrapService() { - super("AlertBootstrapService"); + private final AlertRegistryClient alertRegistryClient; + + private final AlertPluginManager alertPluginManager; + + private final AlertHAServer alertHAServer; + + private final AlertEventFetcher alertEventFetcher; + + private final AlertEventLoop alertEventLoop; + + private final ListenerEventLoop listenerEventLoop; + + private final ListenerEventFetcher listenerEventFetcher; + + public AlertBootstrapService(AlertRpcServer alertRpcServer, + AlertRegistryClient alertRegistryClient, + AlertPluginManager alertPluginManager, + AlertHAServer alertHAServer, + AlertEventFetcher alertEventFetcher, + AlertEventLoop alertEventLoop, + ListenerEventLoop listenerEventLoop, + ListenerEventFetcher listenerEventFetcher) { + this.alertRpcServer = alertRpcServer; + this.alertRegistryClient = alertRegistryClient; + this.alertPluginManager = alertPluginManager; + this.alertHAServer = alertHAServer; + this.alertEventFetcher = alertEventFetcher; + this.alertEventLoop = alertEventLoop; + this.listenerEventLoop = listenerEventLoop; + this.listenerEventFetcher = listenerEventFetcher; } - @Override - public void run() { - log.info("Alert sender thread started"); - while (!ServerLifeCycleManager.isStopped()) { - try { - List alerts = alertDao.listPendingAlerts(); - if (CollectionUtils.isEmpty(alerts)) { - log.debug("There is not waiting alerts"); - continue; - } - AlertServerMetrics.registerPendingAlertGauge(alerts::size); - this.send(alerts); - } catch (Exception e) { - log.error("Alert sender thread meet an exception", e); - } finally { - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 5L); - } - } - log.info("Alert sender thread stopped"); - } + public void start() { + log.info("AlertBootstrapService starting..."); + alertPluginManager.start(); + alertRpcServer.start(); + alertRegistryClient.start(); + alertHAServer.start(); - public void send(List alerts) { - for (Alert alert : alerts) { - // get alert group from alert - int alertId = alert.getId(); - int alertGroupId = Optional.ofNullable(alert.getAlertGroupId()).orElse(0); - List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); - if (CollectionUtils.isEmpty(alertInstanceList)) { - log.error("send alert msg fail,no bind plugin instance."); - List alertResults = Lists.newArrayList(new AlertResult("false", - "no bind plugin instance")); - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, JSONUtils.toJsonString(alertResults), alertId); - continue; - } - AlertData alertData = AlertData.builder() - .id(alertId) - .content(alert.getContent()) - .log(alert.getLog()) - .title(alert.getTitle()) - .warnType(alert.getWarningType().getCode()) - .alertType(alert.getAlertType().getCode()) - .build(); + listenerEventFetcher.start(); + alertEventFetcher.start(); - int sendSuccessCount = 0; - List alertSendStatuses = new ArrayList<>(); - List alertResults = new ArrayList<>(); - for (AlertPluginInstance instance : alertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertStatus sendStatus = Boolean.parseBoolean(alertResult.getStatus()) - ? AlertStatus.EXECUTION_SUCCESS - : AlertStatus.EXECUTION_FAILURE; - AlertSendStatus alertSendStatus = AlertSendStatus.builder() - .alertId(alertId) - .alertPluginInstanceId(instance.getId()) - .sendStatus(sendStatus) - .log(JSONUtils.toJsonString(alertResult)) - .createTime(new Date()) - .build(); - alertSendStatuses.add(alertSendStatus); - if (AlertStatus.EXECUTION_SUCCESS.equals(sendStatus)) { - sendSuccessCount++; - AlertServerMetrics.incAlertSuccessCount(); - } else { - AlertServerMetrics.incAlertFailCount(); - } - alertResults.add(alertResult); - } - } - AlertStatus alertStatus = AlertStatus.EXECUTION_SUCCESS; - if (sendSuccessCount == 0) { - alertStatus = AlertStatus.EXECUTION_FAILURE; - } else if (sendSuccessCount < alertInstanceList.size()) { - alertStatus = AlertStatus.EXECUTION_PARTIAL_SUCCESS; - } - // we update the alert first to avoid duplicate key in alertSendStatus - // this may loss the alertSendStatus if the server restart - // todo: use transaction to update these two table - alertDao.updateAlert(alertStatus, JSONUtils.toJsonString(alertResults), alertId); - alertDao.insertAlertSendStatus(alertSendStatuses); - } - } - - /** - * sync send alert handler - * - * @param alertGroupId alertGroupId - * @param title title - * @param content content - * @return AlertSendResponseCommand - */ - public AlertSendResponse syncHandler(int alertGroupId, String title, String content, int warnType) { - List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); - AlertData alertData = AlertData.builder() - .content(content) - .title(title) - .warnType(warnType) - .build(); - - boolean sendResponseStatus = true; - List sendResponseResults = new ArrayList<>(); - - if (CollectionUtils.isEmpty(alertInstanceList)) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - String message = String.format("Alert GroupId %s send error : not found alert instance", alertGroupId); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(message); - sendResponseResults.add(alertSendResponseResult); - log.error("Alert GroupId {} send error : not found alert instance", alertGroupId); - return new AlertSendResponse(false, sendResponseResults); - } - - for (AlertPluginInstance instance : alertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult( - Boolean.parseBoolean(alertResult.getStatus()), - alertResult.getMessage()); - sendResponseStatus = sendResponseStatus && alertSendResponseResult.isSuccess(); - sendResponseResults.add(alertSendResponseResult); - } - } - - return new AlertSendResponse(sendResponseStatus, sendResponseResults); - } - - /** - * alert result handler - * - * @param instance instance - * @param alertData alertData - * @return AlertResult - */ - private @Nullable AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) { - String pluginInstanceName = instance.getInstanceName(); - int pluginDefineId = instance.getPluginDefineId(); - Optional alertChannelOptional = alertPluginManager.getAlertChannel(instance.getPluginDefineId()); - if (!alertChannelOptional.isPresent()) { - String message = String.format("Alert Plugin %s send error: the channel doesn't exist, pluginDefineId: %s", - pluginInstanceName, - pluginDefineId); - log.error("Alert Plugin {} send error : not found plugin {}", pluginInstanceName, pluginDefineId); - return new AlertResult("false", message); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams()); - String instanceWarnType = WarningType.ALL.getDescp(); - - if (MapUtils.isNotEmpty(paramsMap)) { - instanceWarnType = paramsMap.getOrDefault(AlertConstants.NAME_WARNING_TYPE, WarningType.ALL.getDescp()); - } - - WarningType warningType = WarningType.of(instanceWarnType); - - if (warningType == null) { - String message = String.format("Alert Plugin %s send error : plugin warnType is null", pluginInstanceName); - log.error("Alert Plugin {} send error : plugin warnType is null", pluginInstanceName); - return new AlertResult("false", message); - } - - boolean sendWarning = false; - switch (warningType) { - case ALL: - sendWarning = true; - break; - case SUCCESS: - if (alertData.getWarnType() == WarningType.SUCCESS.getCode()) { - sendWarning = true; - } - break; - case FAILURE: - if (alertData.getWarnType() == WarningType.FAILURE.getCode()) { - sendWarning = true; - } - break; - default: - } - - if (!sendWarning) { - String message = String.format( - "Alert Plugin %s send ignore warning type not match: plugin warning type is %s, alert data warning type is %s", - pluginInstanceName, warningType.getCode(), alertData.getWarnType()); - log.info( - "Alert Plugin {} send ignore warning type not match: plugin warning type is {}, alert data warning type is {}", - pluginInstanceName, warningType.getCode(), alertData.getWarnType()); - return new AlertResult("false", message); - } - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .alertPluginInstanceId(instance.getId()) - .build(); - int waitTimeout = alertConfig.getWaitTimeout(); - try { - AlertResult alertResult; - if (waitTimeout <= 0) { - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - alertResult = alertChannel.closeAlert(alertInfo); - } else { - alertResult = alertChannel.process(alertInfo); - } - } else { - CompletableFuture future; - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); - } else { - future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); - } - alertResult = future.get(waitTimeout, TimeUnit.MILLISECONDS); - } - if (alertResult == null) { - throw new RuntimeException("Alert result cannot be null"); - } - return alertResult; - } catch (InterruptedException e) { - log.error("send alert error alert data id :{},", alertData.getId(), e); - Thread.currentThread().interrupt(); - return new AlertResult("false", e.getMessage()); - } catch (Exception e) { - log.error("send alert error alert data id :{},", alertData.getId(), e); - return new AlertResult("false", e.getMessage()); - } - } - - public AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams) { - - boolean sendResponseStatus = true; - List sendResponseResults = new ArrayList<>(); - - Optional alertChannelOptional = alertPluginManager.getAlertChannel(pluginDefineId); - if (!alertChannelOptional.isPresent()) { - String message = String.format("Test send alert error: the channel doesn't exist, pluginDefineId: %s", - pluginDefineId); - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(message); - sendResponseResults.add(alertSendResponseResult); - log.error("Test send alert error : not found plugin {}", pluginDefineId); - return new AlertSendResponse(false, sendResponseResults); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = PluginParamsTransfer.getPluginParamsMap(pluginInstanceParams); - - AlertData alertData = AlertData.builder() - .title(AlertConstants.TEST_TITLE) - .content(AlertConstants.TEST_CONTENT) - .warnType(WarningType.ALL.getCode()) - .build(); - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .build(); - - try { - AlertResult alertResult = alertChannel.process(alertInfo); - if (alertResult != null) { - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult( - Boolean.parseBoolean(alertResult.getStatus()), - alertResult.getMessage()); - sendResponseStatus = alertSendResponseResult.isSuccess(); - sendResponseResults.add(alertSendResponseResult); - } - } catch (Exception e) { - log.error("Test send alert error", e); - AlertSendResponse.AlertSendResponseResult alertSendResponseResult = - new AlertSendResponse.AlertSendResponseResult(); - alertSendResponseResult.setSuccess(false); - alertSendResponseResult.setMessage(e.getMessage()); - sendResponseResults.add(alertSendResponseResult); - return new AlertSendResponse(false, sendResponseResults); - } - - return new AlertSendResponse(sendResponseStatus, sendResponseResults); + listenerEventLoop.start(); + alertEventLoop.start(); + log.info("AlertBootstrapService started..."); } @Override public void close() { - log.info("Closed AlertBootstrapService..."); - } + log.info("AlertBootstrapService stopping..."); + try ( + AlertRpcServer closedAlertRpcServer = alertRpcServer; + AlertRegistryClient closedAlertRegistryClient = alertRegistryClient) { + // close resource + listenerEventFetcher.shutdown(); + alertEventFetcher.shutdown(); + listenerEventLoop.shutdown(); + alertEventLoop.shutdown(); + alertHAServer.shutdown(); + } + log.info("AlertBootstrapService stopped..."); + } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java new file mode 100644 index 0000000000..11a668ae1d --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventFetcher.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertEventFetcher extends AbstractEventFetcher { + + private final AlertDao alertDao; + + public AlertEventFetcher(AlertHAServer alertHAServer, + AlertDao alertDao, + AlertEventPendingQueue alertEventPendingQueue) { + super("AlertEventFetcher", alertHAServer, alertEventPendingQueue); + this.alertDao = alertDao; + } + + @Override + public List fetchPendingEvent(int eventOffset) { + return alertDao.listPendingAlerts(eventOffset); + } + + @Override + protected int getEventOffset(Alert event) { + return event.getId(); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java new file mode 100644 index 0000000000..e975f1ad51 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventLoop.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertEventLoop extends AbstractEventLoop { + + private final AlertSender alertSender; + + public AlertEventLoop(AlertEventPendingQueue alertEventPendingQueue, + AlertSenderThreadPoolFactory alertSenderThreadPoolFactory, + AlertSender alertSender) { + super("AlertEventLoop", alertSenderThreadPoolFactory.getThreadPool(), alertEventPendingQueue); + this.alertSender = alertSender; + AlertServerMetrics.registerPendingAlertGauge(this::getHandlingEventCount); + } + + @Override + public void handleEvent(Alert event) { + alertSender.sendEvent(event); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java new file mode 100644 index 0000000000..17fe7ccd0b --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueue.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import org.springframework.stereotype.Component; + +@Component +public class AlertEventPendingQueue extends AbstractEventPendingQueue { + + public AlertEventPendingQueue(AlertConfig alertConfig) { + super(alertConfig.getSenderParallelism() * 3 + 1); + AlertServerMetrics.registerPendingAlertGauge(this::size); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java new file mode 100644 index 0000000000..998bc655c4 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertHAServer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; +import org.apache.dolphinscheduler.registry.api.ha.AbstractHAServer; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertHAServer extends AbstractHAServer { + + public AlertHAServer(Registry registry) { + super(registry, RegistryNodeType.ALERT_LOCK.getRegistryPath()); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java new file mode 100644 index 0000000000..9c9cd034bd --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AlertSender extends AbstractEventSender { + + private final AlertDao alertDao; + + public AlertSender(AlertDao alertDao, + AlertPluginManager alertPluginManager, + AlertConfig alertConfig) { + super(alertPluginManager, alertConfig.getWaitTimeout()); + this.alertDao = alertDao; + } + + /** + * sync send alert handler + * + * @param alertGroupId alertGroupId + * @param title title + * @param content content + * @return AlertSendResponseCommand + */ + public AlertSendResponse syncHandler(int alertGroupId, String title, String content) { + List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); + AlertData alertData = AlertData.builder() + .content(content) + .title(title) + .build(); + + boolean sendResponseStatus = true; + List sendResponseResults = new ArrayList<>(); + + if (CollectionUtils.isEmpty(alertInstanceList)) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + new AlertSendResponse.AlertSendResponseResult(); + String message = String.format("Alert GroupId %s send error : not found alert instance", alertGroupId); + alertSendResponseResult.setSuccess(false); + alertSendResponseResult.setMessage(message); + sendResponseResults.add(alertSendResponseResult); + log.error("Alert GroupId {} send error : not found alert instance", alertGroupId); + return new AlertSendResponse(false, sendResponseResults); + } + + for (AlertPluginInstance instance : alertInstanceList) { + AlertResult alertResult = doSendEvent(instance, alertData); + if (alertResult != null) { + AlertSendResponse.AlertSendResponseResult alertSendResponseResult = + new AlertSendResponse.AlertSendResponseResult( + alertResult.isSuccess(), + alertResult.getMessage()); + sendResponseStatus = sendResponseStatus && alertSendResponseResult.isSuccess(); + sendResponseResults.add(alertSendResponseResult); + } + } + + return new AlertSendResponse(sendResponseStatus, sendResponseResults); + } + + @Override + public List getAlertPluginInstanceList(Alert event) { + return alertDao.listInstanceByAlertGroupId(event.getAlertGroupId()); + } + + @Override + public AlertData getAlertData(Alert event) { + return AlertData.builder() + .id(event.getId()) + .content(event.getContent()) + .log(event.getLog()) + .title(event.getTitle()) + .alertType(event.getAlertType().getCode()) + .build(); + } + + @Override + public Integer getEventId(Alert event) { + return event.getId(); + } + + @Override + public void onError(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, log, event.getId()); + } + + @Override + public void onPartialSuccess(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_PARTIAL_SUCCESS, log, event.getId()); + } + + @Override + public void onSuccess(Alert event, String log) { + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, log, event.getId()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java new file mode 100644 index 0000000000..fd8c731b17 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.springframework.stereotype.Component; + +@Component +public class AlertSenderThreadPoolFactory { + + private final ThreadPoolExecutor threadPool; + + public AlertSenderThreadPoolFactory(AlertConfig alertConfig) { + this.threadPool = ThreadUtils.newDaemonFixedThreadExecutor("AlertSenderThread", + alertConfig.getSenderParallelism()); + } + + public ThreadPoolExecutor getThreadPool() { + return threadPool; + } + +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java similarity index 61% rename from dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java index df88de34e3..089fb4edc9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventFetcher.java @@ -15,22 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.alert.service; -import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; - -import org.mockito.Mockito; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; +import java.util.List; /** - * dependency config + * The interface responsible for fetching events. + * + * @param the type of event */ -@Configuration -public class TaskResponseProcessorTestConfig { +public interface EventFetcher { - @Bean - public DataQualityResultOperator dataQualityResultOperator() { - return Mockito.mock(DataQualityResultOperator.class); - } + void start(); + + List fetchPendingEvent(int eventOffset); + + void shutdown(); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java new file mode 100644 index 0000000000..04219f99ae --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventLoop.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +/** + * The interface responsible for consuming event from upstream, e.g {@link EventPendingQueue}. + * + * @param the type of event + */ +public interface EventLoop { + + /** + * Start the event loop, once the event loop is started, it will keep consuming event from upstream. + */ + void start(); + + /** + * Handle the given event. + */ + void handleEvent(T event); + + /** + * Get the count of handling event. + */ + int getHandlingEventCount(); + + /** + * Shutdown the event loop, once the event loop is shutdown, it will stop consuming event from upstream. + */ + void shutdown(); + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java new file mode 100644 index 0000000000..c8538138bc --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventPendingQueue.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +/** + * The interface responsible for managing pending events. + * + * @param the type of event + */ +public interface EventPendingQueue { + + void put(T alert) throws InterruptedException; + + T take() throws InterruptedException; + + int size(); + + int capacity(); +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java new file mode 100644 index 0000000000..04bc85e573 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/EventSender.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; + +public interface EventSender { + + void sendEvent(T event); + + AlertResult doSendEvent(AlertPluginInstance instance, AlertData alertData); + + AlertSendResponse syncTestSend(int pluginDefineId, String pluginInstanceParams); + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java new file mode 100644 index 0000000000..57549d7e97 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventFetcher.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class ListenerEventFetcher extends AbstractEventFetcher { + + private final ListenerEventDao listenerEventDao; + + protected ListenerEventFetcher(AlertHAServer alertHAServer, + ListenerEventDao listenerEventDao, + ListenerEventPendingQueue listenerEventPendingQueue) { + super("ListenerEventFetcher", alertHAServer, listenerEventPendingQueue); + this.listenerEventDao = listenerEventDao; + } + + @Override + protected int getEventOffset(ListenerEvent event) { + return event.getId(); + } + + @Override + public List fetchPendingEvent(int eventOffset) { + return listenerEventDao.listingPendingEvents(eventOffset, FETCH_SIZE); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java new file mode 100644 index 0000000000..f1c00967f9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventLoop.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import org.springframework.stereotype.Component; + +@Component +public class ListenerEventLoop extends AbstractEventLoop { + + private final ListenerEventSender listenerEventSender; + + protected ListenerEventLoop(AlertSenderThreadPoolFactory alertSenderThreadPoolFactory, + ListenerEventSender listenerEventSender, + ListenerEventPendingQueue listenerEventPendingQueue) { + super("ListenerEventLoop", alertSenderThreadPoolFactory.getThreadPool(), listenerEventPendingQueue); + this.listenerEventSender = listenerEventSender; + } + + @Override + public void handleEvent(ListenerEvent event) { + listenerEventSender.sendEvent(event); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java new file mode 100644 index 0000000000..47d0c77dff --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPendingQueue.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import org.springframework.stereotype.Component; + +@Component +public class ListenerEventPendingQueue extends AbstractEventPendingQueue { + + public ListenerEventPendingQueue(AlertConfig alertConfig) { + super(alertConfig.getSenderParallelism() * 3 + 1); + } + +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java deleted file mode 100644 index b57562c711..0000000000 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventPostService.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.alert.service; - -import org.apache.dolphinscheduler.alert.api.AlertChannel; -import org.apache.dolphinscheduler.alert.api.AlertData; -import org.apache.dolphinscheduler.alert.api.AlertInfo; -import org.apache.dolphinscheduler.alert.api.AlertResult; -import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; -import org.apache.dolphinscheduler.dao.entity.ListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.AbstractListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionCreatedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionDeletedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionUpdatedListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessEndListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessFailListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ProcessStartListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskEndListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskFailListenerEvent; -import org.apache.dolphinscheduler.dao.entity.event.TaskStartListenerEvent; -import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.curator.shaded.com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nullable; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -@Service -@Slf4j -public final class ListenerEventPostService extends BaseDaemonThread implements AutoCloseable { - - @Value("${alert.query_alert_threshold:100}") - private Integer QUERY_ALERT_THRESHOLD; - @Autowired - private ListenerEventMapper listenerEventMapper; - @Autowired - private AlertPluginInstanceMapper alertPluginInstanceMapper; - @Autowired - private AlertPluginManager alertPluginManager; - @Autowired - private AlertConfig alertConfig; - - public ListenerEventPostService() { - super("ListenerEventPostService"); - } - - @Override - public void run() { - log.info("listener event post thread started"); - while (!ServerLifeCycleManager.isStopped()) { - try { - List listenerEvents = listenerEventMapper - .listingListenerEventByStatus(AlertStatus.WAIT_EXECUTION, QUERY_ALERT_THRESHOLD); - if (CollectionUtils.isEmpty(listenerEvents)) { - log.debug("There is no waiting listener events"); - continue; - } - this.send(listenerEvents); - } catch (Exception e) { - log.error("listener event post thread meet an exception", e); - } finally { - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 5L); - } - } - log.info("listener event post thread stopped"); - } - - public void send(List listenerEvents) { - for (ListenerEvent listenerEvent : listenerEvents) { - int eventId = listenerEvent.getId(); - List globalAlertInstanceList = - alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList(); - if (CollectionUtils.isEmpty(globalAlertInstanceList)) { - log.error("post listener event fail,no bind global plugin instance."); - listenerEventMapper.updateListenerEvent(eventId, AlertStatus.EXECUTION_FAILURE, - "no bind plugin instance", new Date()); - continue; - } - AbstractListenerEvent event = generateEventFromContent(listenerEvent); - if (event == null) { - log.error("parse listener event to abstract listener event fail.ed {}", listenerEvent.getContent()); - listenerEventMapper.updateListenerEvent(eventId, AlertStatus.EXECUTION_FAILURE, - "parse listener event to abstract listener event failed", new Date()); - continue; - } - List events = Lists.newArrayList(event); - AlertData alertData = AlertData.builder() - .id(eventId) - .content(JSONUtils.toJsonString(events)) - .log(listenerEvent.getLog()) - .title(event.getTitle()) - .warnType(WarningType.GLOBAL.getCode()) - .alertType(event.getEventType().getCode()) - .build(); - - int sendSuccessCount = 0; - List failedPostResults = new ArrayList<>(); - for (AlertPluginInstance instance : globalAlertInstanceList) { - AlertResult alertResult = this.alertResultHandler(instance, alertData); - if (alertResult != null) { - AlertStatus sendStatus = Boolean.parseBoolean(alertResult.getStatus()) - ? AlertStatus.EXECUTION_SUCCESS - : AlertStatus.EXECUTION_FAILURE; - if (AlertStatus.EXECUTION_SUCCESS.equals(sendStatus)) { - sendSuccessCount++; - } else { - AlertSendStatus alertSendStatus = AlertSendStatus.builder() - .alertId(eventId) - .alertPluginInstanceId(instance.getId()) - .sendStatus(sendStatus) - .log(JSONUtils.toJsonString(alertResult)) - .createTime(new Date()) - .build(); - failedPostResults.add(alertSendStatus); - } - } - } - if (sendSuccessCount == globalAlertInstanceList.size()) { - listenerEventMapper.deleteById(eventId); - } else { - AlertStatus alertStatus = - sendSuccessCount == 0 ? AlertStatus.EXECUTION_FAILURE : AlertStatus.EXECUTION_PARTIAL_SUCCESS; - listenerEventMapper.updateListenerEvent(eventId, alertStatus, JSONUtils.toJsonString(failedPostResults), - new Date()); - } - } - } - - /** - * alert result handler - * - * @param instance instance - * @param alertData alertData - * @return AlertResult - */ - private @Nullable AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) { - String pluginInstanceName = instance.getInstanceName(); - int pluginDefineId = instance.getPluginDefineId(); - Optional alertChannelOptional = alertPluginManager.getAlertChannel(instance.getPluginDefineId()); - if (!alertChannelOptional.isPresent()) { - String message = - String.format("Global Alert Plugin %s send error: the channel doesn't exist, pluginDefineId: %s", - pluginInstanceName, - pluginDefineId); - log.error("Global Alert Plugin {} send error : not found plugin {}", pluginInstanceName, pluginDefineId); - return new AlertResult("false", message); - } - AlertChannel alertChannel = alertChannelOptional.get(); - - Map paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams()); - - AlertInfo alertInfo = AlertInfo.builder() - .alertData(alertData) - .alertParams(paramsMap) - .alertPluginInstanceId(instance.getId()) - .build(); - int waitTimeout = alertConfig.getWaitTimeout(); - try { - AlertResult alertResult; - if (waitTimeout <= 0) { - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - alertResult = alertChannel.closeAlert(alertInfo); - } else { - alertResult = alertChannel.process(alertInfo); - } - } else { - CompletableFuture future; - if (alertData.getAlertType() == AlertType.CLOSE_ALERT.getCode()) { - future = CompletableFuture.supplyAsync(() -> alertChannel.closeAlert(alertInfo)); - } else { - future = CompletableFuture.supplyAsync(() -> alertChannel.process(alertInfo)); - } - alertResult = future.get(waitTimeout, TimeUnit.MILLISECONDS); - } - if (alertResult == null) { - throw new RuntimeException("Alert result cannot be null"); - } - return alertResult; - } catch (InterruptedException e) { - log.error("post listener event error alert data id :{},", alertData.getId(), e); - Thread.currentThread().interrupt(); - return new AlertResult("false", e.getMessage()); - } catch (Exception e) { - log.error("post listener event error alert data id :{},", alertData.getId(), e); - return new AlertResult("false", e.getMessage()); - } - } - - private AbstractListenerEvent generateEventFromContent(ListenerEvent listenerEvent) { - String content = listenerEvent.getContent(); - switch (listenerEvent.getEventType()) { - case SERVER_DOWN: - return JSONUtils.parseObject(content, ServerDownListenerEvent.class); - case PROCESS_DEFINITION_CREATED: - return JSONUtils.parseObject(content, ProcessDefinitionCreatedListenerEvent.class); - case PROCESS_DEFINITION_UPDATED: - return JSONUtils.parseObject(content, ProcessDefinitionUpdatedListenerEvent.class); - case PROCESS_DEFINITION_DELETED: - return JSONUtils.parseObject(content, ProcessDefinitionDeletedListenerEvent.class); - case PROCESS_START: - return JSONUtils.parseObject(content, ProcessStartListenerEvent.class); - case PROCESS_END: - return JSONUtils.parseObject(content, ProcessEndListenerEvent.class); - case PROCESS_FAIL: - return JSONUtils.parseObject(content, ProcessFailListenerEvent.class); - case TASK_START: - return JSONUtils.parseObject(content, TaskStartListenerEvent.class); - case TASK_END: - return JSONUtils.parseObject(content, TaskEndListenerEvent.class); - case TASK_FAIL: - return JSONUtils.parseObject(content, TaskFailListenerEvent.class); - default: - return null; - } - } - @Override - public void close() { - log.info("Closed ListenerEventPostService..."); - } -} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java new file mode 100644 index 0000000000..7d06500edd --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/ListenerEventSender.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import org.apache.dolphinscheduler.alert.api.AlertData; +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.AbstractListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionCreatedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionDeletedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessDefinitionUpdatedListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessEndListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessFailListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ProcessStartListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskEndListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskFailListenerEvent; +import org.apache.dolphinscheduler.dao.entity.event.TaskStartListenerEvent; +import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import org.apache.curator.shaded.com.google.common.collect.Lists; + +import java.util.Date; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class ListenerEventSender extends AbstractEventSender { + + private final ListenerEventDao listenerEventDao; + + private final AlertPluginInstanceMapper alertPluginInstanceMapper; + + public ListenerEventSender(ListenerEventDao listenerEventDao, + AlertPluginInstanceMapper alertPluginInstanceMapper, + AlertPluginManager alertPluginManager, + AlertConfig alertConfig) { + super(alertPluginManager, alertConfig.getWaitTimeout()); + this.listenerEventDao = listenerEventDao; + this.alertPluginInstanceMapper = alertPluginInstanceMapper; + } + + private AbstractListenerEvent generateEventFromContent(ListenerEvent listenerEvent) { + String content = listenerEvent.getContent(); + AbstractListenerEvent event = null; + switch (listenerEvent.getEventType()) { + case SERVER_DOWN: + event = JSONUtils.parseObject(content, ServerDownListenerEvent.class); + break; + case PROCESS_DEFINITION_CREATED: + event = JSONUtils.parseObject(content, ProcessDefinitionCreatedListenerEvent.class); + break; + case PROCESS_DEFINITION_UPDATED: + event = JSONUtils.parseObject(content, ProcessDefinitionUpdatedListenerEvent.class); + break; + case PROCESS_DEFINITION_DELETED: + event = JSONUtils.parseObject(content, ProcessDefinitionDeletedListenerEvent.class); + break; + case PROCESS_START: + event = JSONUtils.parseObject(content, ProcessStartListenerEvent.class); + break; + case PROCESS_END: + event = JSONUtils.parseObject(content, ProcessEndListenerEvent.class); + break; + case PROCESS_FAIL: + event = JSONUtils.parseObject(content, ProcessFailListenerEvent.class); + break; + case TASK_START: + event = JSONUtils.parseObject(content, TaskStartListenerEvent.class); + break; + case TASK_END: + event = JSONUtils.parseObject(content, TaskEndListenerEvent.class); + break; + case TASK_FAIL: + event = JSONUtils.parseObject(content, TaskFailListenerEvent.class); + break; + default: + throw new IllegalArgumentException("Unsupported event type: " + listenerEvent.getEventType()); + } + if (event == null) { + throw new IllegalArgumentException("Failed to parse event from content: " + content); + } + return event; + } + + @Override + public List getAlertPluginInstanceList(ListenerEvent event) { + return alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList(); + } + + @Override + public AlertData getAlertData(ListenerEvent listenerEvent) { + AbstractListenerEvent event = generateEventFromContent(listenerEvent); + return AlertData.builder() + .id(listenerEvent.getId()) + .content(JSONUtils.toJsonString(Lists.newArrayList(event))) + .log(listenerEvent.getLog()) + .title(event.getTitle()) + .alertType(event.getEventType().getCode()) + .build(); + } + + @Override + public Integer getEventId(ListenerEvent event) { + return event.getId(); + } + + @Override + public void onError(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_FAILURE, log, new Date()); + } + + @Override + public void onPartialSuccess(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_PARTIAL_SUCCESS, log, new Date()); + } + + @Override + public void onSuccess(ListenerEvent event, String log) { + listenerEventDao.updateListenerEvent(event.getId(), AlertStatus.EXECUTION_FAILURE, log, new Date()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 0dbb6988ce..6fbcc04feb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -73,7 +73,8 @@ alert: # Define value is (0 = infinite), and alert server would be waiting alert result. wait-timeout: 0 max-heartbeat-interval: 60s - query_alert_threshold: 100 + # The maximum number of alerts that can be processed in parallel + sender-parallelism: 100 registry: type: zookeeper diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java new file mode 100644 index 0000000000..1a72f0a5c9 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/config/AlertConfigTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.config; + +import static com.google.common.truth.Truth.assertThat; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; + +@AutoConfigureMockMvc +@SpringBootTest(classes = AlertConfig.class) +class AlertConfigTest { + + @Autowired + private AlertConfig alertConfig; + + @Test + void testValidate() { + assertThat(alertConfig.getWaitTimeout()).isEqualTo(10); + assertThat(alertConfig.getMaxHeartbeatInterval()).isEqualTo(Duration.ofSeconds(59)); + assertThat(alertConfig.getSenderParallelism()).isEqualTo(101); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java similarity index 64% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java index 5e757c7858..75f16848fd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java @@ -15,19 +15,24 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.event; +package org.apache.dolphinscheduler.alert.rpc; -/** - * This exception represent the exception can be recovered, when we get this exception, - * we will move the event to the fail of the queue. - */ -public class StateEventHandleFailure extends Exception { +import org.apache.dolphinscheduler.alert.config.AlertConfig; - public StateEventHandleFailure(String message) { - super(message); +import org.junit.jupiter.api.Test; + +class AlertRpcServerTest { + + private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig()); + + @Test + void testStart() { + alertRpcServer.start(); } - public StateEventHandleFailure(String message, Throwable throwable) { - super(message, throwable); + @Test + void testClose() { + alertRpcServer.close(); } + } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java similarity index 73% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index eafba16585..400afd34dc 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -24,15 +24,13 @@ import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; +import org.apache.dolphinscheduler.alert.service.AlertSender; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; -import org.apache.dolphinscheduler.dao.entity.ListenerEvent; -import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; @@ -42,19 +40,20 @@ import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AlertBootstrapServiceTest { +@ExtendWith(MockitoExtension.class) +class AlertSenderTest { - private static final Logger logger = LoggerFactory.getLogger(AlertBootstrapServiceTest.class); + private static final Logger logger = LoggerFactory.getLogger(AlertSenderTest.class); @Mock private AlertDao alertDao; @@ -66,7 +65,7 @@ public class AlertBootstrapServiceTest { private AlertConfig alertConfig; @InjectMocks - private AlertBootstrapService alertBootstrapService; + private AlertSender alertSender; private static final String PLUGIN_INSTANCE_PARAMS = "{\"User\":\"xx\",\"receivers\":\"xx\",\"sender\":\"xx\",\"smtpSslTrust\":\"*\",\"enableSmtpAuth\":\"true\",\"receiverCcs\":null,\"showType\":\"table\",\"starttlsEnable\":\"false\",\"serverPort\":\"25\",\"serverHost\":\"xx\",\"Password\":\"xx\",\"sslEnable\":\"false\"}"; @@ -74,25 +73,17 @@ public class AlertBootstrapServiceTest { private static final String PLUGIN_INSTANCE_NAME = "alert-instance-mail"; private static final String TITLE = "alert mail test TITLE"; private static final String CONTENT = "alert mail test CONTENT"; - private static final List EVENTS = new ArrayList<>(); private static final int PLUGIN_DEFINE_ID = 1; private static final int ALERT_GROUP_ID = 1; - @BeforeEach - public void before() { - MockitoAnnotations.initMocks(this); - } - @Test - public void testSyncHandler() { + void testSyncHandler() { // 1.alert instance does not exist when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(null); - when(alertConfig.getWaitTimeout()).thenReturn(0); - AlertSendResponse alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + AlertSendResponse alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -108,12 +99,7 @@ public class AlertBootstrapServiceTest { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList); - String pluginName = "alert-plugin-mail"; - PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null); - when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); - - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -122,37 +108,32 @@ public class AlertBootstrapServiceTest { AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(null); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - when(alertConfig.getWaitTimeout()).thenReturn(0); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); // 4.abnormal information inside the alert plug-in code AlertResult alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(false)); + alertResult.setSuccess(false); alertResult.setMessage("Abnormal information inside the alert plug-in code"); when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); // 5.alert plugin send success alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(true)); + alertResult.setSuccess(true); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - when(alertConfig.getWaitTimeout()).thenReturn(5000); - alertSendResponse = - alertBootstrapService.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, WarningType.ALL.getCode()); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertTrue(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -160,17 +141,13 @@ public class AlertBootstrapServiceTest { } @Test - public void testRun() { - List alertList = new ArrayList<>(); + void testRun() { Alert alert = new Alert(); alert.setId(1); alert.setAlertGroupId(ALERT_GROUP_ID); alert.setTitle(TITLE); alert.setContent(CONTENT); alert.setWarningType(WarningType.FAILURE); - alertList.add(alert); - - // alertSenderService = new AlertSenderService(); int pluginDefineId = 1; String pluginInstanceParams = "alert-instance-mail-params"; @@ -181,25 +158,18 @@ public class AlertBootstrapServiceTest { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(alertInstanceList); - String pluginName = "alert-plugin-mail"; - PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null); - when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); - AlertResult alertResult = new AlertResult(); - alertResult.setStatus(String.valueOf(true)); + alertResult.setSuccess(true); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); - AlertChannel alertChannelMock = mock(AlertChannel.class); - when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); - when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertTrue(Boolean.parseBoolean(alertResult.getStatus())); + Assertions.assertTrue(alertResult.isSuccess()); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(new ArrayList<>()); - alertBootstrapService.send(alertList); + alertSender.sendEvent(alert); } @Test - public void testSendAlert() { + void testSendAlert() { AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(true)); + sendResult.setSuccess(true); sendResult.setMessage(String.format("Alert Plugin %s send success", PLUGIN_INSTANCE_NAME)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); @@ -209,6 +179,6 @@ public class AlertBootstrapServiceTest { Mockito.mockStatic(PluginParamsTransfer.class); pluginParamsTransferMockedStatic.when(() -> PluginParamsTransfer.getPluginParamsMap(PLUGIN_INSTANCE_PARAMS)) .thenReturn(paramsMap); - alertBootstrapService.syncTestSend(PLUGIN_DEFINE_ID, PLUGIN_INSTANCE_PARAMS); + alertSender.syncTestSend(PLUGIN_DEFINE_ID, PLUGIN_INSTANCE_PARAMS); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java similarity index 82% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java index 33917267f0..0304be022c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventPostServiceTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/ListenerEventSenderTest.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; +import org.apache.dolphinscheduler.alert.service.ListenerEventSender; import org.apache.dolphinscheduler.common.enums.AlertPluginInstanceType; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.ListenerEventType; @@ -33,7 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.ListenerEvent; import org.apache.dolphinscheduler.dao.entity.event.ServerDownListenerEvent; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; import org.apache.commons.codec.digest.DigestUtils; @@ -43,39 +43,32 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.mockito.junit.jupiter.MockitoExtension; -public class ListenerEventPostServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(ListenerEventPostServiceTest.class); +@ExtendWith(MockitoExtension.class) +class ListenerEventSenderTest { @Mock - private ListenerEventMapper listenerEventMapper; + private ListenerEventDao listenerEventDao; + @Mock private AlertPluginInstanceMapper alertPluginInstanceMapper; @Mock private AlertPluginManager alertPluginManager; + @Mock private AlertConfig alertConfig; @InjectMocks - private ListenerEventPostService listenerEventPostService; - - @BeforeEach - public void before() { - MockitoAnnotations.initMocks(this); - } + private ListenerEventSender listenerEventSender; @Test - public void testSendServerDownEventSuccess() { - List events = new ArrayList<>(); + void testSendServerDownEventSuccess() { ServerDownListenerEvent serverDownListenerEvent = new ServerDownListenerEvent(); serverDownListenerEvent.setEventTime(new Date()); serverDownListenerEvent.setType("WORKER"); @@ -88,7 +81,6 @@ public class ListenerEventPostServiceTest { successEvent.setEventType(ListenerEventType.SERVER_DOWN); successEvent.setCreateTime(new Date()); successEvent.setUpdateTime(new Date()); - events.add(successEvent); int pluginDefineId = 1; String pluginInstanceParams = @@ -103,19 +95,17 @@ public class ListenerEventPostServiceTest { when(alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList()).thenReturn(alertInstanceList); AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(true)); + sendResult.setSuccess(true); sendResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertTrue(Boolean.parseBoolean(sendResult.getStatus())); - when(listenerEventMapper.deleteById(1)).thenReturn(1); - listenerEventPostService.send(events); + Assertions.assertTrue(sendResult.isSuccess()); + listenerEventSender.sendEvent(successEvent); } @Test - public void testSendServerDownEventFailed() { - List events = new ArrayList<>(); + void testSendServerDownEventFailed() { ServerDownListenerEvent serverDownListenerEvent = new ServerDownListenerEvent(); serverDownListenerEvent.setEventTime(new Date()); serverDownListenerEvent.setType("WORKER"); @@ -128,7 +118,6 @@ public class ListenerEventPostServiceTest { successEvent.setEventType(ListenerEventType.SERVER_DOWN); successEvent.setCreateTime(new Date()); successEvent.setUpdateTime(new Date()); - events.add(successEvent); int pluginDefineId = 1; String pluginInstanceParams = @@ -143,12 +132,12 @@ public class ListenerEventPostServiceTest { when(alertPluginInstanceMapper.queryAllGlobalAlertPluginInstanceList()).thenReturn(alertInstanceList); AlertResult sendResult = new AlertResult(); - sendResult.setStatus(String.valueOf(false)); + sendResult.setSuccess(false); sendResult.setMessage(String.format("Alert Plugin %s send failed", pluginInstanceName)); AlertChannel alertChannelMock = mock(AlertChannel.class); when(alertChannelMock.process(Mockito.any())).thenReturn(sendResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - Assertions.assertFalse(Boolean.parseBoolean(sendResult.getStatus())); - listenerEventPostService.send(events); + Assertions.assertFalse(sendResult.isSuccess()); + listenerEventSender.sendEvent(successEvent); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java new file mode 100644 index 0000000000..a643e50e76 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertEventPendingQueueTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.truth.Truth.assertThat; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.dao.entity.Alert; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +import lombok.SneakyThrows; + +import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AlertEventPendingQueueTest { + + private AlertEventPendingQueue alertEventPendingQueue; + + private static final int QUEUE_SIZE = 10; + + @BeforeEach + public void before() { + AlertConfig alertConfig = new AlertConfig(); + alertConfig.setSenderParallelism(QUEUE_SIZE); + this.alertEventPendingQueue = new AlertEventPendingQueue(alertConfig); + } + + @SneakyThrows + @Test + void put() { + for (int i = 0; i < alertEventPendingQueue.capacity(); i++) { + alertEventPendingQueue.put(new Alert()); + } + + CompletableFuture completableFuture = CompletableFuture.runAsync(() -> { + try { + alertEventPendingQueue.put(new Alert()); + System.out.println(alertEventPendingQueue.size()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + assertThrowsExactly(ConditionTimeoutException.class, + () -> await() + .timeout(Duration.ofSeconds(2)) + .until(completableFuture::isDone)); + + } + + @Test + void take() { + CompletableFuture completableFuture = CompletableFuture.runAsync(() -> { + try { + alertEventPendingQueue.take(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + assertThrowsExactly(ConditionTimeoutException.class, + () -> await() + .timeout(Duration.ofSeconds(2)) + .until(completableFuture::isDone)); + } + + @SneakyThrows + @Test + void size() { + for (int i = 0; i < alertEventPendingQueue.capacity(); i++) { + alertEventPendingQueue.put(new Alert()); + assertThat(alertEventPendingQueue.size()).isEqualTo(i + 1); + } + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java new file mode 100644 index 0000000000..50f44fb43f --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/service/AlertSenderThreadPoolFactoryTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.service; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.alert.config.AlertConfig; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AlertSenderThreadPoolFactoryTest { + + private final AlertConfig alertConfig = new AlertConfig(); + + private final AlertSenderThreadPoolFactory alertSenderThreadPoolFactory = + new AlertSenderThreadPoolFactory(alertConfig); + + @Test + void getThreadPool() { + ThreadPoolExecutor threadPool = alertSenderThreadPoolFactory.getThreadPool(); + assertThat(threadPool.getCorePoolSize()).isEqualTo(alertConfig.getSenderParallelism()); + assertThat(threadPool.getMaximumPoolSize()).isEqualTo(alertConfig.getSenderParallelism()); + } +} diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml new file mode 100644 index 0000000000..d16d05a678 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/application.yaml @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + profiles: + active: postgresql + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + banner: + charset: UTF-8 + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + +server: + port: 50053 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +alert: + port: 50052 + # Mark each alert of alert server if late after x milliseconds as failed. + # Define value is (0 = infinite), and alert server would be waiting alert result. + wait-timeout: 10 + max-heartbeat-interval: 59s + # The maximum number of alerts that can be processed in parallel + sender-parallelism: 101 + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ + +metrics: + enabled: true + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/logback.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index c9fdab4d51..5453f8fb15 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -57,6 +57,13 @@ conf + + ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources + + **/*.yaml + + conf + ${basedir}/../dolphinscheduler-ui/dist ./ui diff --git a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh index c668944139..d953e04d2f 100644 --- a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh @@ -24,6 +24,7 @@ -XX:+PrintGCDetails -Xloggc:gc.log +-XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 6f7d8f43d2..f8b705179a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -69,6 +70,7 @@ public class ApiApplicationServer { log.info("Received spring application context ready event will load taskPlugin and write to DB"); // install task plugin TaskPluginManager.loadPlugin(); + DataSourceProcessorProvider.initialize(); for (Map.Entry entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { String taskPluginName = entry.getKey(); TaskChannelFactory taskChannelFactory = entry.getValue(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java index 8dc628b576..1b6283e0dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -126,7 +126,7 @@ public class OperatorUtils { return auditType.getAuditOperationType(); } - public static long getObjectIdentityByParma(String[] paramNameArr, Map paramsMap) { + public static long getObjectIdentityByParam(String[] paramNameArr, Map paramsMap) { for (String name : paramNameArr) { if (paramsMap.get(name) instanceof String) { String param = (String) paramsMap.get(name); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java index 0ab607da65..1e34ff2967 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -80,14 +80,14 @@ public abstract class BaseAuditOperator implements AuditOperator { } modifyRequestParams(paramNameArr, paramsMap, auditLogList); - setObjectByParma(paramNameArr, paramsMap, auditLogList); + setObjectByParam(paramNameArr, paramsMap, auditLogList); if (auditLogList.get(0).getModelId() == null) { - auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParma(paramNameArr, paramsMap)); + auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParam(paramNameArr, paramsMap)); } } - protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + protected void setObjectByParam(String[] paramNameArr, Map paramsMap, List auditLogList) { String name = paramNameArr[0]; @@ -97,7 +97,7 @@ public abstract class BaseAuditOperator implements AuditOperator { return; } - String objName = getObjectNameFromReturnIdentity(value); + String objName = getObjectNameFromIdentity(value); if (Strings.isNullOrEmpty(objName)) { auditLogList.get(0).setModelName(value.toString()); @@ -114,7 +114,7 @@ public abstract class BaseAuditOperator implements AuditOperator { auditLogList.get(0).setModelName(objName); } - protected void setObjectByParmaArr(String[] paramNameArr, Map paramsMap, + protected void setObjectByParamArr(String[] paramNameArr, Map paramsMap, List auditLogList) { AuditLog auditLog = auditLogList.get(0); @@ -127,7 +127,7 @@ public abstract class BaseAuditOperator implements AuditOperator { for (String identityString : identityArr) { long identity = toLong(identityString); - String value = getObjectNameFromReturnIdentity(identity); + String value = getObjectNameFromIdentity(identity); if (value == null) { continue; @@ -156,7 +156,7 @@ public abstract class BaseAuditOperator implements AuditOperator { protected void setObjectNameFromReturnIdentity(List auditLogList) { auditLogList - .forEach(auditLog -> auditLog.setModelName(getObjectNameFromReturnIdentity(auditLog.getModelId()))); + .forEach(auditLog -> auditLog.setModelName(getObjectNameFromIdentity(auditLog.getModelId()))); } protected void modifyObjectFromReturnObject(String[] params, Map returnObjectMap, @@ -180,7 +180,7 @@ public abstract class BaseAuditOperator implements AuditOperator { return NumberUtils.toLong(str.toString(), -1); } - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { return identity.toString(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java index d9d55b71f7..65f0f491be 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class AlertGroupAuditOperatorImpl extends BaseAuditOperator { private AlertGroupMapper alertGroupMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java index 428007a22b..d2c28832f0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class AlertInstanceAuditOperatorImpl extends BaseAuditOperator { private AlertPluginInstanceMapper alertPluginInstanceMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java index 41cf81fb40..4f1cabc043 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class ClusterAuditOperatorImpl extends BaseAuditOperator { private ClusterMapper clusterMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java index 7a58cce19c..a859ab8cea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class DatasourceAuditOperatorImpl extends BaseAuditOperator { private DataSourceMapper dataSourceMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java index fc8301b048..cf8a93e366 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class EnvironmentAuditOperatorImpl extends BaseAuditOperator { private EnvironmentMapper environmentMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java index e8bf25ef2d..9f0fe6b4e5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class K8SNamespaceAuditOperatorImpl extends BaseAuditOperator { private K8sNamespaceMapper k8sNamespaceMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java index ba87229729..5a65adbe55 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java @@ -46,14 +46,14 @@ public class ProcessAuditOperatorImpl extends BaseAuditOperator { } @Override - protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + protected void setObjectByParam(String[] paramNameArr, Map paramsMap, List auditLogList) { if (paramNameArr[0].equals(AuditLogConstants.CODES) || paramNameArr[0].equals(AuditLogConstants.PROCESS_DEFINITION_CODES) || paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) { - super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList); + super.setObjectByParamArr(paramNameArr, paramsMap, auditLogList); } else { - super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + super.setObjectByParam(paramNameArr, paramsMap, auditLogList); } if (paramsMap.containsKey(AuditLogConstants.VERSION)) { if (paramsMap.get(AuditLogConstants.VERSION) != null) { @@ -65,7 +65,7 @@ public class ProcessAuditOperatorImpl extends BaseAuditOperator { } @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java index f77ac59733..cb45d5abba 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java @@ -48,17 +48,17 @@ public class ProcessInstanceAuditOperatorImpl extends BaseAuditOperator { } @Override - protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + protected void setObjectByParam(String[] paramNameArr, Map paramsMap, List auditLogList) { if (paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) { - super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList); + super.setObjectByParamArr(paramNameArr, paramsMap, auditLogList); } else { - super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + super.setObjectByParam(paramNameArr, paramsMap, auditLogList); } } @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { int objId = NumberUtils.toInt(identity.toString(), -1); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java index 186d9df4e1..62cea94f95 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java @@ -34,7 +34,7 @@ public class ProjectAuditOperatorImpl extends BaseAuditOperator { private ProjectMapper projectMapper; @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java index f7326bca21..303f7bfdb3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java @@ -37,7 +37,7 @@ public class ResourceAuditOperatorImpl extends BaseAuditOperator { } @Override - protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + protected void setObjectByParam(String[] paramNameArr, Map paramsMap, List auditLogList) { Object objName = getFileNameFromParam(paramNameArr, paramsMap); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java index 731eaa25c7..be2ea935ed 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java @@ -72,7 +72,7 @@ public class ScheduleAuditOperatorImpl extends BaseAuditOperator { } @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java index 202a56cce9..db6559b865 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java @@ -46,17 +46,17 @@ public class TaskAuditOperatorImpl extends BaseAuditOperator { } @Override - protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + protected void setObjectByParam(String[] paramNameArr, Map paramsMap, List auditLogList) { - super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + super.setObjectByParam(paramNameArr, paramsMap, auditLogList); if (paramsMap.containsKey(AuditLogConstants.VERSION)) { auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString()); } } @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java index f50fef2e8d..12e42890f3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class TaskGroupAuditOperatorImpl extends BaseAuditOperator { private TaskGroupMapper taskGroupMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java index 1b8067fb0b..8d888b7e57 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class TaskInstancesAuditOperatorImpl extends BaseAuditOperator { private TaskInstanceMapper taskInstanceMapper; @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java index 4dd80bc422..f76ddd4124 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class TenantAuditOperatorImpl extends BaseAuditOperator { private TenantMapper tenantMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java index 39c5a4f2f7..96f1964996 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java @@ -54,7 +54,7 @@ public class TokenAuditOperatorImpl extends BaseAuditOperator { } @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java index 9a185621df..2e7b9ae274 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java @@ -33,7 +33,7 @@ public class UdfFunctionAuditOperatorImpl extends BaseAuditOperator { private UdfFuncMapper udfFuncMapper; @Override - protected String getObjectNameFromReturnIdentity(Object identity) { + protected String getObjectNameFromIdentity(Object identity) { int objId = NumberUtils.toInt(identity.toString(), -1); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java index 88db6923cd..bbce39cb02 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class UserAuditOperatorImpl extends BaseAuditOperator { private UserMapper userMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java index e576a64474..f7a2642904 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java @@ -48,7 +48,7 @@ public class WorkerGroupAuditOperatorImpl extends BaseAuditOperator { } @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java index cf7608bb61..25b15f09e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java @@ -31,7 +31,7 @@ public class YarnQueueAuditOperatorImpl extends BaseAuditOperator { private QueueMapper queueMapper; @Override - public String getObjectNameFromReturnIdentity(Object identity) { + public String getObjectNameFromIdentity(Object identity) { Long objId = toLong(identity); if (objId == -1) { return ""; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java index c69c1d969c..9e57ce34dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java @@ -20,17 +20,21 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.COMMAND_STATE_COUNT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.COUNT_PROCESS_DEFINITION_USER_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.COUNT_PROCESS_INSTANCE_STATE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.LIST_PAGING_ALERT_GROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUEUE_COUNT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.TASK_INSTANCE_STATE_COUNT_ERROR; import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataAnalysisService; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.vo.TaskInstanceCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowDefinitionCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowInstanceCountVO; import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; import org.apache.dolphinscheduler.dao.entity.User; import java.util.List; @@ -148,4 +152,54 @@ public class DataAnalysisController extends BaseController { Map stringIntegerMap = dataAnalysisService.countQueueState(loginUser); return Result.success(stringIntegerMap); } + + /** + * command queue + * + * @param loginUser login user + * @return queue state count + */ + @Operation(summary = "listPendingCommands", description = "LIST_PENDING_COMMANDS") + @Parameters({ + @Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)), + @Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")), + @Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20")) + }) + @GetMapping("/listCommand") + @ResponseStatus(HttpStatus.OK) + @ApiException(LIST_PAGING_ALERT_GROUP_ERROR) + public Result> listPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "projectCode", required = false) Long projectCode, + @RequestParam("pageNo") Integer pageNo, + @RequestParam("pageSize") Integer pageSize) { + checkPageParams(pageNo, pageSize); + PageInfo commandPageInfo = + dataAnalysisService.listPendingCommands(loginUser, projectCode, pageNo, pageSize); + return Result.success(commandPageInfo); + } + + /** + * error command + * + * @param loginUser login user + * @return queue state count + */ + @Operation(summary = "listErrorCommand", description = "LIST_ERROR_COMMAND_LIST_PAGING_NOTES") + @Parameters({ + @Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)), + @Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")), + @Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20")) + }) + @GetMapping("/listErrorCommand") + @ResponseStatus(HttpStatus.OK) + @ApiException(LIST_PAGING_ALERT_GROUP_ERROR) + public Result> listErrorCommand(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "projectCode", required = false) Long projectCode, + @RequestParam("pageNo") Integer pageNo, + @RequestParam("pageSize") Integer pageSize) { + checkPageParams(pageNo, pageSize); + PageInfo errorCommandPageInfo = + dataAnalysisService.listErrorCommand(loginUser, projectCode, pageNo, pageSize); + return Result.success(errorCommandPageInfo); + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index a2a300b946..8c86de7fe6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -44,6 +44,8 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertyUtils; import org.apache.commons.lang3.StringUtils; @@ -163,10 +165,8 @@ public class ExecutorController extends BaseController { if (timeout == null) { timeout = Constants.MAX_TASK_TIMEOUT; } - Map startParamMap = null; - if (startParams != null) { - startParamMap = JSONUtils.toMap(startParams); - } + + List startParamList = PropertyUtils.startParamsTransformPropertyList(startParams); if (complementDependentMode == null) { complementDependentMode = ComplementDependentMode.OFF_MODE; @@ -175,7 +175,7 @@ public class ExecutorController extends BaseController { Map result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, + workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber, dryRun, testFlag, complementDependentMode, version, allLevelDependent, executionOrder); return returnDataList(result); @@ -262,10 +262,7 @@ public class ExecutorController extends BaseController { timeout = Constants.MAX_TASK_TIMEOUT; } - Map startParamMap = null; - if (startParams != null) { - startParamMap = JSONUtils.toMap(startParams); - } + List startParamList = PropertyUtils.startParamsTransformPropertyList(startParams); if (complementDependentMode == null) { log.debug("Parameter complementDependentMode set to {} due to null.", ComplementDependentMode.OFF_MODE); @@ -283,7 +280,8 @@ public class ExecutorController extends BaseController { result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, tenantCode, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, + workerGroup, tenantCode, environmentCode, timeout, startParamList, expectedParallelismNumber, + dryRun, testFlag, complementDependentMode, null, allLevelDependent, executionOrder); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java index 7c0a8ef0e1..4bf952d006 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.LIST_MASTERS_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.LIST_WORKERS_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATABASE_STATE_ERROR; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -26,15 +25,16 @@ import org.apache.dolphinscheduler.api.service.MonitorService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.model.WorkerServerModel; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseStatus; @@ -56,35 +56,19 @@ public class MonitorController extends BaseController { private MonitorService monitorService; /** - * master list + * server list * - * @param loginUser login user - * @return master list + * @return server list */ - @Operation(summary = "listMaster", description = "MASTER_LIST_NOTES") - @GetMapping(value = "/masters") + @Operation(summary = "listServer", description = "SERVER_LIST_NOTES") + @GetMapping(value = "/{nodeType}") @ResponseStatus(HttpStatus.OK) @ApiException(LIST_MASTERS_ERROR) - public Result> listMaster(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { - List servers = monitorService.queryMaster(loginUser); + public Result> listServer(@PathVariable("nodeType") RegistryNodeType nodeType) { + List servers = monitorService.listServer(nodeType); return Result.success(servers); } - /** - * worker list - * - * @param loginUser login user - * @return worker information list - */ - @Operation(summary = "listWorker", description = "WORKER_LIST_NOTES") - @GetMapping(value = "/workers") - @ResponseStatus(HttpStatus.OK) - @ApiException(LIST_WORKERS_ERROR) - public Result> listWorker(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { - List workerServerModels = monitorService.queryWorker(loginUser); - return Result.success(workerServerModels); - } - /** * query database state * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java index 706c7e939a..efec970e6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectParameterController.java @@ -61,7 +61,8 @@ public class ProjectParameterController extends BaseController { @Operation(summary = "createProjectParameter", description = "CREATE_PROJECT_PARAMETER_NOTES") @Parameters({ @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)), - @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)) + @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)), + @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class)) }) @PostMapping() @ResponseStatus(HttpStatus.CREATED) @@ -69,9 +70,10 @@ public class ProjectParameterController extends BaseController { public Result createProjectParameter(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("projectParameterName") String projectParameterName, - @RequestParam(value = "projectParameterValue") String projectParameterValue) { + @RequestParam(value = "projectParameterValue") String projectParameterValue, + @RequestParam(value = "projectParameterDataType", defaultValue = "VARCHAR") String projectParameterDataType) { return projectParameterService.createProjectParameter(loginUser, projectCode, projectParameterName, - projectParameterValue); + projectParameterValue, projectParameterDataType); } @Operation(summary = "updateProjectParameter", description = "UPDATE_PROJECT_PARAMETER_NOTES") @@ -79,6 +81,7 @@ public class ProjectParameterController extends BaseController { @Parameter(name = "code", description = "PROJECT_PARAMETER_CODE", schema = @Schema(implementation = long.class, example = "123456")), @Parameter(name = "projectParameterName", description = "PROJECT_PARAMETER_NAME", schema = @Schema(implementation = String.class)), @Parameter(name = "projectParameterValue", description = "PROJECT_PARAMETER_VALUE", schema = @Schema(implementation = String.class)), + @Parameter(name = "projectParameterDataType", description = "PROJECT_PARAMETER_DATA_TYPE", schema = @Schema(implementation = String.class)) }) @PutMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @@ -87,9 +90,10 @@ public class ProjectParameterController extends BaseController { @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("code") Long code, @RequestParam("projectParameterName") String projectParameterName, - @RequestParam(value = "projectParameterValue") String projectParameterValue) { + @RequestParam(value = "projectParameterValue") String projectParameterValue, + @RequestParam(value = "projectParameterDataType") String projectParameterDataType) { return projectParameterService.updateProjectParameter(loginUser, projectCode, code, projectParameterName, - projectParameterValue); + projectParameterValue, projectParameterDataType); } @Operation(summary = "deleteProjectParametersByCode", description = "DELETE_PROJECT_PARAMETER_NOTES") @@ -133,13 +137,14 @@ public class ProjectParameterController extends BaseController { @Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "searchVal", required = false) String searchVal, + @RequestParam(value = "projectParameterDataType", required = false) String projectParameterDataType, @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize) { checkPageParams(pageNo, pageSize); searchVal = ParameterUtils.handleEscapes(searchVal); return projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, - searchVal); + searchVal, projectParameterDataType); } @Operation(summary = "queryProjectParameterByCode", description = "QUERY_PROJECT_PARAMETER_NOTES") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java index 3c978795ab..9ab2d29f6a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java @@ -44,7 +44,7 @@ public class ApiExceptionHandler { @ExceptionHandler(Throwable.class) public Result exceptionHandler(Throwable e, HandlerMethod hm) { ApiException ce = hm.getMethodAnnotation(ApiException.class); - log.error("Meet en unknown exception: ", e); + log.error("Meet an unknown exception: ", e); if (ce == null) { return Result.errorWithArgs(Status.INTERNAL_SERVER_ERROR_ARGS, e.getMessage()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index 351a3d5c91..f312707c2c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -21,9 +21,12 @@ import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.dto.DefineUserDto; import org.apache.dolphinscheduler.api.dto.TaskCountDto; import org.apache.dolphinscheduler.api.dto.project.StatisticsStateRequest; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.vo.TaskInstanceCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowDefinitionCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowInstanceCountVO; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; import org.apache.dolphinscheduler.dao.entity.User; import java.util.List; @@ -117,4 +120,7 @@ public interface DataAnalysisService { */ TaskCountDto countOneTaskStates(User loginUser, Long taskCode); + PageInfo listPendingCommands(User loginUser, Long projectCode, Integer pageNo, Integer pageSize); + + PageInfo listErrorCommand(User loginUser, Long projectCode, Integer pageNo, Integer pageSize); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 3166d3e718..55ca697412 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -30,7 +30,9 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import java.util.List; import java.util.Map; /** @@ -57,7 +59,7 @@ public interface ExecutorService { * @param environmentCode environment code * @param runMode run mode * @param timeout timeout - * @param startParams the global param values which pass to new process instance + * @param startParamList the global param values which pass to new process instance * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode * @param executionOrder the execution order when complementing data * @return execute process instance code @@ -71,7 +73,7 @@ public interface ExecutorService { Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, Integer timeout, - Map startParams, Integer expectedParallelismNumber, + List startParamList, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, Integer version, boolean allLevelDependent, ExecutionOrder executionOrder); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index c4a8d94a1f..f054fafdce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.model.WorkerServerModel; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import java.util.List; @@ -38,20 +38,10 @@ public interface MonitorService { List queryDatabaseState(User loginUser); /** - * query master list + * query server list * - * @param loginUser login user - * @return master information list + * @param nodeType RegistryNodeType + * @return server information list */ - List queryMaster(User loginUser); - - /** - * query worker list - * - * @param loginUser login user - * @return worker information list - */ - List queryWorker(User loginUser); - - List getServerListFromRegistry(boolean isMaster); + List listServer(RegistryNodeType nodeType); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java index ef984dfac4..af2b047488 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectParameterService.java @@ -23,17 +23,17 @@ import org.apache.dolphinscheduler.dao.entity.User; public interface ProjectParameterService { Result createProjectParameter(User loginUser, long projectCode, String projectParameterName, - String projectParameterValue); + String projectParameterValue, String projectParameterDataType); Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName, - String projectParameterValue); + String projectParameterValue, String projectParameterDataType); Result deleteProjectParametersByCode(User loginUser, long projectCode, long code); Result batchDeleteProjectParametersByCodes(User loginUser, long projectCode, String codes); Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo, - String searchVal); + String searchVal, String projectParameterDataType); Result queryProjectParameterByCode(User loginUser, long projectCode, long code); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java index 36c9c2b8c1..9520b5090c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -27,14 +27,18 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.DataAnalysisService; import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.vo.TaskInstanceCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowDefinitionCountVO; import org.apache.dolphinscheduler.api.vo.WorkflowInstanceCountVO; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; @@ -71,6 +75,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.collect.Lists; /** @@ -380,6 +386,66 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal return new TaskCountDto(executeStatusCounts); } + @Override + public PageInfo listPendingCommands(User loginUser, Long projectCode, Integer pageNo, Integer pageSize) { + Page page = new Page<>(pageNo, pageSize); + if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { + IPage commandIPage = commandMapper.queryCommandPage(page); + return PageInfo.of(commandIPage); + } + + List workflowDefinitionCodes = getAuthDefinitionCodes(loginUser, projectCode); + + if (workflowDefinitionCodes.isEmpty()) { + return PageInfo.of(pageNo, pageSize); + } + + IPage commandIPage = + commandMapper.queryCommandPageByIds(page, new ArrayList<>(workflowDefinitionCodes)); + return PageInfo.of(commandIPage); + } + + @Override + public PageInfo listErrorCommand(User loginUser, Long projectCode, Integer pageNo, Integer pageSize) { + Page page = new Page<>(pageNo, pageSize); + if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { + IPage commandIPage = errorCommandMapper.queryErrorCommandPage(page); + return PageInfo.of(commandIPage); + } + + List workflowDefinitionCodes = getAuthDefinitionCodes(loginUser, projectCode); + + if (workflowDefinitionCodes.isEmpty()) { + return PageInfo.of(pageNo, pageSize); + } + + IPage commandIPage = + errorCommandMapper.queryErrorCommandPageByIds(page, new ArrayList<>(workflowDefinitionCodes)); + return PageInfo.of(commandIPage); + } + + private List getAuthDefinitionCodes(User loginUser, Long projectCode) { + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), log); + if (CollectionUtils.isEmpty(projectIds)) { + return Collections.emptyList(); + } + List projectCodes = projectMapper.selectBatchIds(projectIds) + .stream() + .map(Project::getCode) + .collect(Collectors.toList()); + + if (projectCode != null) { + if (!projectCodes.contains(projectCode)) { + return Collections.emptyList(); + } + + projectCodes = Collections.singletonList(projectCode); + } + + return processDefinitionMapper.queryDefinitionCodeListByProjectCodes(projectCodes); + } + /** * statistics the process definition quantities of a certain person *

diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 5b576ce95a..3e613b11fa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -90,6 +90,8 @@ import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTrigg import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; @@ -203,7 +205,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ * @param environmentCode environment code * @param runMode run mode * @param timeout timeout - * @param startParams the global param values which pass to new process instance + * @param startParamList the global param values which pass to new process instance * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode * @param testFlag testFlag * @param executionOrder the execution order when complementing data @@ -219,7 +221,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, Integer timeout, - Map startParams, Integer expectedParallelismNumber, + List startParamList, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, Integer version, boolean allLevelDependent, ExecutionOrder executionOrder) { @@ -269,7 +271,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ startNodeList, cronTime, warningType, loginUser.getId(), warningGroupId, runMode, processInstancePriority, workerGroup, tenantCode, - environmentCode, startParams, expectedParallelismNumber, dryRun, testFlag, + environmentCode, startParamList, expectedParallelismNumber, dryRun, testFlag, complementDependentMode, allLevelDependent, executionOrder); if (create > 0) { @@ -289,7 +291,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ private void checkMasterExists() { // check master server exists - List masterServers = monitorService.getServerListFromRegistry(true); + List masterServers = monitorService.listServer(RegistryNodeType.MASTER); // no master if (masterServers.isEmpty()) { @@ -731,7 +733,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ WarningType warningType, int executorId, Integer warningGroupId, RunMode runMode, Priority processInstancePriority, String workerGroup, String tenantCode, Long environmentCode, - Map startParams, Integer expectedParallelismNumber, int dryRun, + List startParamList, Integer expectedParallelismNumber, int dryRun, int testFlag, ComplementDependentMode complementDependentMode, boolean allLevelDependent, ExecutionOrder executionOrder) { @@ -760,8 +762,8 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ if (warningType != null) { command.setWarningType(warningType); } - if (startParams != null && startParams.size() > 0) { - cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParams)); + if (CollectionUtils.isNotEmpty(startParamList)) { + cmdParam.put(CMD_PARAM_START_PARAMS, JSONUtils.toJsonString(startParamList)); } command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setExecutorId(executorId); @@ -1142,7 +1144,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ checkValidTenant(tenantCode); checkMasterExists(); // todo dispatch improvement - List masterServerList = monitorService.getServerListFromRegistry(true); + List masterServerList = monitorService.listServer(RegistryNodeType.MASTER); Server server = masterServerList.get(0); StreamingTaskTriggerRequest taskExecuteStartMessage = new StreamingTaskTriggerRequest(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java index 907ab2329c..24bfcc5b25 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.service.MonitorService; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.model.WorkerServerModel; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; @@ -27,7 +26,6 @@ import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import java.util.List; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -35,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; /** * monitor service impl @@ -61,48 +58,8 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic return Lists.newArrayList(databaseMonitor.getDatabaseMetrics()); } - /** - * query master list - * - * @param loginUser login user - * @return master information list - */ @Override - public List queryMaster(User loginUser) { - return registryClient.getServerList(RegistryNodeType.MASTER); + public List listServer(RegistryNodeType nodeType) { + return registryClient.getServerList(nodeType); } - - /** - * query worker list - * - * @param loginUser login user - * @return worker information list - */ - @Override - public List queryWorker(User loginUser) { - - return registryClient.getServerList(RegistryNodeType.WORKER) - .stream() - .map((Server server) -> { - WorkerServerModel model = new WorkerServerModel(); - model.setId(server.getId()); - model.setHost(server.getHost()); - model.setPort(server.getPort()); - model.setZkDirectories(Sets.newHashSet(server.getZkDirectory())); - model.setResInfo(server.getResInfo()); - model.setCreateTime(server.getCreateTime()); - model.setLastHeartbeatTime(server.getLastHeartbeatTime()); - return model; - }) - .collect(Collectors.toList()); - - } - - @Override - public List getServerListFromRegistry(boolean isMaster) { - return isMaster - ? registryClient.getServerList(RegistryNodeType.MASTER) - : registryClient.getServerList(RegistryNodeType.WORKER); - } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index 17d1d04712..e23101da36 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -68,7 +68,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override @Transactional public Result createProjectParameter(User loginUser, long projectCode, String projectParameterName, - String projectParameterValue) { + String projectParameterValue, String projectParameterDataType) { Result result = new Result(); // check if user have write perm for project @@ -97,6 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj .builder() .paramName(projectParameterName) .paramValue(projectParameterValue) + .paramDataType(projectParameterDataType) .code(CodeGenerateUtils.genCode()) .projectCode(projectCode) .userId(loginUser.getId()) @@ -122,7 +123,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override public Result updateProjectParameter(User loginUser, long projectCode, long code, String projectParameterName, - String projectParameterValue) { + String projectParameterValue, String projectParameterDataType) { Result result = new Result(); // check if user have write perm for project @@ -155,6 +156,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj projectParameter.setParamName(projectParameterName); projectParameter.setParamValue(projectParameterValue); + projectParameter.setParamDataType(projectParameterDataType); projectParameter.setUpdateTime(new Date()); projectParameter.setOperator(loginUser.getId()); @@ -236,7 +238,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj @Override public Result queryProjectParameterListPaging(User loginUser, long projectCode, Integer pageSize, Integer pageNo, - String searchVal) { + String searchVal, String projectParameterDataType) { Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); @@ -249,7 +251,8 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj Page page = new Page<>(pageNo, pageSize); IPage iPage = - projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal); + projectParameterMapper.queryProjectParameterListPaging(page, projectCode, null, searchVal, + projectParameterDataType); List projectParameterList = iPage.getRecords(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 1c039cdfbd..6a15da17a8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkFilePath; import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS; import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; @@ -1291,10 +1290,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { return; } - // abnormal characters check - if (!checkFilePath(fullName)) { - throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH); - } // Avoid returning to the parent directory if (fullName.contains("../")) { throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index 7469d8db13..49e42da561 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -367,10 +367,15 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst } for (TaskInstance taskInstance : needToDeleteTaskInstances) { if (StringUtils.isNotBlank(taskInstance.getLogPath())) { - ILogService iLogService = - SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(), - ILogService.class); - iLogService.removeTaskInstanceLog(taskInstance.getLogPath()); + try { + // Remove task instance log failed will not affect the deletion of task instance + ILogService iLogService = + SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(), + ILogService.class); + iLogService.removeTaskInstanceLog(taskInstance.getLogPath()); + } catch (Exception ex) { + log.error("Remove task instance log error", ex); + } } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index b394d4956c..8b166a16dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -158,14 +158,4 @@ public class CheckUtils { return pattern.matcher(str).matches(); } - - /** - * regex FilePath check,only use a to z, A to Z, 0 to 9, and _./- - * - * @param str input string - * @return true if regex pattern is right, otherwise return false - */ - public static boolean checkFilePath(String str) { - return regexChecks(str, Constants.REGEX_FILE_PATH); - } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java index 2e9961b5d2..36622b8b98 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java @@ -150,4 +150,40 @@ public class DataAnalysisControllerTest extends AbstractControllerTest { assertThat(result.getCode().intValue()).isEqualTo(Status.SUCCESS.getCode()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void testListCommand() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("projectCode", "16"); + paramsMap.add("pageNo", "1"); + paramsMap.add("pageSize", "10"); + + MvcResult mvcResult = mockMvc.perform(get("/projects/analysis/listCommand") + .header("sessionId", sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + assertThat(result.getCode().intValue()).isEqualTo(Status.SUCCESS.getCode()); + logger.info(mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testListErrorCommand() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("projectCode", "16"); + paramsMap.add("pageNo", "1"); + paramsMap.add("pageSize", "10"); + + MvcResult mvcResult = mockMvc.perform(get("/projects/analysis/listErrorCommand") + .header("sessionId", sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + assertThat(result.getCode().intValue()).isEqualTo(Status.SUCCESS.getCode()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java index 2c1a62ccc6..5b77acc099 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java @@ -38,8 +38,13 @@ import org.apache.dolphinscheduler.common.enums.RunMode; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -75,7 +80,8 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { final String tenantCode = "root"; final Long environmentCode = 4L; final Integer timeout = 5; - final ImmutableMap startParams = ImmutableMap.of("start", "params"); + final List startParams = + Collections.singletonList(new Property("start", Direct.IN, DataType.VARCHAR, "params")); final Integer expectedParallelismNumber = 6; final int dryRun = 7; final int testFlag = 0; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 737faa1636..93dc48b80c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -41,8 +41,7 @@ public class MonitorControllerTest extends AbstractControllerTest { @Test public void testListMaster() throws Exception { - - MvcResult mvcResult = mockMvc.perform(get("/monitor/masters") + MvcResult mvcResult = mockMvc.perform(get("/monitor/MASTER") .header(SESSION_ID, sessionId) /* .param("type", ResourceType.FILE.name()) */) .andExpect(status().isOk()) @@ -59,7 +58,24 @@ public class MonitorControllerTest extends AbstractControllerTest { @Test public void testListWorker() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/monitor/workers") + MvcResult mvcResult = mockMvc.perform(get("/monitor/WORKER") + .header(SESSION_ID, sessionId) + /* .param("type", ResourceType.FILE.name()) */) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + result.getCode().equals(Status.SUCCESS.getCode()); + + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testListAlert() throws Exception { + + MvcResult mvcResult = mockMvc.perform(get("/monitor/ALERT_SERVER") .header(SESSION_ID, sessionId) /* .param("type", ResourceType.FILE.name()) */) .andExpect(status().isOk()) @@ -76,8 +92,7 @@ public class MonitorControllerTest extends AbstractControllerTest { @Test public void testQueryDatabaseState() throws Exception { MvcResult mvcResult = mockMvc.perform(get("/monitor/databases") - .header(SESSION_ID, sessionId) - /* .param("type", ResourceType.FILE.name()) */) + .header(SESSION_ID, sessionId)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java index f1bd20bb23..ea8bbf67fd 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectParameterControllerTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -48,8 +49,9 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.createProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.any(), - Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value"); + Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.createProjectParameter(loginUser, 1, "key", "value", + DataType.VARCHAR.name()); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -58,8 +60,9 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.updateProjectParameter(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), - Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value"); + Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.updateProjectParameter(loginUser, 1, 1L, "key", "value", + DataType.LONG.name()); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -88,8 +91,9 @@ public class ProjectParameterControllerTest { User loginUser = getGeneralUser(); Mockito.when(projectParameterService.queryProjectParameterListPaging(Mockito.any(), Mockito.anyLong(), - Mockito.anyInt(), Mockito.anyInt(), Mockito.any())).thenReturn(getSuccessResult()); - Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1", 1, 10); + Mockito.anyInt(), Mockito.anyInt(), Mockito.any(), Mockito.any())).thenReturn(getSuccessResult()); + Result result = projectParameterController.queryProjectParameterListPaging(loginUser, 1, "1", + DataType.VARCHAR.name(), 1, 10); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 32033cc2af..a2d655f47f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -32,12 +32,15 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.vo.TaskInstanceCountVO; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.User; @@ -70,6 +73,9 @@ import org.mockito.quality.Strictness; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * data analysis service test */ @@ -110,13 +116,17 @@ public class DataAnalysisServiceTest { private User user; + private Project project; + @BeforeEach public void setUp() { user = new User(); + user.setUserType(UserType.ADMIN_USER); user.setId(1); - Project project = new Project(); + project = new Project(); project.setId(1); + project.setCode(1); project.setName("test"); resultMap = new HashMap<>(); @@ -285,6 +295,66 @@ public class DataAnalysisServiceTest { .allMatch(count -> count.equals(0)); } + @Test + public void testListPendingCommands() { + IPage page = new Page<>(1, 10); + page.setTotal(2L); + page.setRecords(getList()); + when(commandMapper.queryCommandPage(any())).thenReturn(page); + PageInfo list1 = dataAnalysisServiceImpl.listPendingCommands(user, 1L, 1, 10); + assertThat(list1.getTotal()).isEqualTo(2); + user.setUserType(UserType.GENERAL_USER); + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, 1, + serviceLogger)) + .thenReturn(projectIds()); + PageInfo list2 = dataAnalysisServiceImpl.listPendingCommands(user, 1L, 1, 10); + assertThat(list2.getTotal()).isEqualTo(0); + when(projectMapper.selectBatchIds(any())).thenReturn(Collections.singletonList(project)); + when(processDefinitionMapper.queryDefinitionCodeListByProjectCodes(any())) + .thenReturn(Collections.singletonList(1L)); + when(commandMapper.queryCommandPageByIds(any(), any())).thenReturn(page); + PageInfo list3 = dataAnalysisServiceImpl.listPendingCommands(user, 1L, 1, 10); + assertThat(list3.getTotal()).isEqualTo(2); + } + + @Test + public void testListErrorCommand() { + IPage page = new Page<>(1, 10); + page.setTotal(2L); + page.setRecords(getErrorList()); + when(errorCommandMapper.queryErrorCommandPage(any())).thenReturn(page); + PageInfo list1 = dataAnalysisServiceImpl.listErrorCommand(user, 1L, 1, 10); + assertThat(list1.getTotal()).isEqualTo(2); + user.setUserType(UserType.GENERAL_USER); + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, 1, + serviceLogger)) + .thenReturn(projectIds()); + PageInfo list2 = dataAnalysisServiceImpl.listErrorCommand(user, 1L, 1, 10); + assertThat(list2.getTotal()).isEqualTo(0); + when(projectMapper.selectBatchIds(any())).thenReturn(Collections.singletonList(project)); + when(processDefinitionMapper.queryDefinitionCodeListByProjectCodes(any())) + .thenReturn(Collections.singletonList(1L)); + when(errorCommandMapper.queryErrorCommandPageByIds(any(), any())).thenReturn(page); + PageInfo list3 = dataAnalysisServiceImpl.listErrorCommand(user, 1L, 1, 10); + assertThat(list3.getTotal()).isEqualTo(2); + } + + private List getList() { + List commandList = new ArrayList<>(); + Command command = new Command(); + command.setId(1); + commandList.add(command); + return commandList; + } + + private List getErrorList() { + List commandList = new ArrayList<>(); + ErrorCommand command = new ErrorCommand(); + command.setId(1); + commandList.add(command); + return commandList; + } + private Set projectIds() { Set projectIds = new HashSet<>(); projectIds.add(1); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java index 23085cb3f5..8f1869c1fb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java @@ -67,6 +67,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.TriggerRelationService; @@ -242,7 +243,7 @@ public class ExecuteFunctionServiceTest { Mockito.when(processService.getTenantForProcess(tenantCode, userId)).thenReturn(tenantCode); doReturn(1).when(commandService).createCommand(argThat(c -> c.getId() == null)); doReturn(0).when(commandService).createCommand(argThat(c -> c.getId() != null)); - Mockito.when(monitorService.getServerListFromRegistry(true)).thenReturn(getMasterServersList()); + Mockito.when(monitorService.listServer(RegistryNodeType.MASTER)).thenReturn(getMasterServersList()); Mockito.when(processService.findProcessInstanceDetailById(processInstanceId)) .thenReturn(Optional.ofNullable(processInstance)); Mockito.when(processService.findProcessDefinition(1L, 1)).thenReturn(this.processDefinition); @@ -498,7 +499,7 @@ public class ExecuteFunctionServiceTest { @Test public void testNoMasterServers() { - Mockito.when(monitorService.getServerListFromRegistry(true)).thenReturn(new ArrayList<>()); + Mockito.when(monitorService.listServer(RegistryNodeType.MASTER)).thenReturn(new ArrayList<>()); Assertions.assertThrows(ServiceException.class, () -> executorService.execProcessInstance( loginUser, diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 4861e1004e..972092602f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -40,7 +40,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.extract.common.ILogService; @@ -91,7 +90,7 @@ public class LoggerServiceTest { @Mock private TaskDefinitionMapper taskDefinitionMapper; - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private int nettyServerPort = 18080; @@ -103,11 +102,10 @@ public class LoggerServiceTest { return; } - nettyRemotingServer = new NettyRemotingServer(NettyServerConfig.builder().listenPort(nettyServerPort).build()); - nettyRemotingServer.start(); - SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery = - new SpringServerMethodInvokerDiscovery(nettyRemotingServer); - springServerMethodInvokerDiscovery.postProcessAfterInitialization(new ILogService() { + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( + NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build()); + springServerMethodInvokerDiscovery.start(); + springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { @Override public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { @@ -142,13 +140,14 @@ public class LoggerServiceTest { public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { } - }, "iLogServiceImpl"); + }); + springServerMethodInvokerDiscovery.start(); } @AfterEach public void tearDown() { - if (nettyRemotingServer != null) { - nettyRemotingServer.close(); + if (springServerMethodInvokerDiscovery != null) { + springServerMethodInvokerDiscovery.close(); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java index 6bfbffd093..e1318e936d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -98,13 +98,13 @@ public class MonitorServiceTest { public void testQueryMaster() { mockPermissionCheck(ApiFuncIdentificationConstant.MONITOR_MASTER_VIEW, true); Mockito.when(registryClient.getServerList(RegistryNodeType.MASTER)).thenReturn(getServerList()); - assertDoesNotThrow(() -> monitorService.queryMaster(user)); + assertDoesNotThrow(() -> monitorService.listServer(RegistryNodeType.MASTER)); } @Test public void testQueryWorker() { Mockito.when(registryClient.getServerList(RegistryNodeType.WORKER)).thenReturn(getServerList()); - AssertionsHelper.assertDoesNotThrow(() -> monitorService.queryWorker(user)); + AssertionsHelper.assertDoesNotThrow(() -> monitorService.listServer(RegistryNodeType.WORKER)); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index ca51690ce6..ef8ffd2768 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.ProjectParameter; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import java.util.Collections; @@ -76,7 +77,8 @@ public class ProjectParameterServiceTest { // PERMISSION DENIED when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(false); - Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", + DataType.VARCHAR.name()); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -88,25 +90,29 @@ public class ProjectParameterServiceTest { try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", + DataType.VARCHAR.name()); assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); } // PROJECT_PARAMETER_ALREADY_EXISTS when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value", + DataType.VARCHAR.name()); assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); // INSERT DATA ERROR when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value", + DataType.VARCHAR.name()); assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); - result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value", + DataType.VARCHAR.name()); assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @@ -117,7 +123,8 @@ public class ProjectParameterServiceTest { // NO PERMISSION when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(false); - Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", + DataType.VARCHAR.name()); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -127,28 +134,33 @@ public class ProjectParameterServiceTest { when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", + DataType.VARCHAR.name()); assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // PROJECT_PARAMETER_ALREADY_EXISTS when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value", + DataType.VARCHAR.name()); assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); // PROJECT_UPDATE_ERROR when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value", + DataType.VARCHAR.name()); assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); - result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value", + DataType.LONG.name()); assertEquals(Status.SUCCESS.getCode(), result.getCode()); ProjectParameter projectParameter = (ProjectParameter) result.getData(); assertNotNull(projectParameter.getOperator()); assertNotNull(projectParameter.getUpdateTime()); + assertEquals(DataType.LONG.name(), projectParameter.getParamDataType()); } @Test @@ -223,7 +235,8 @@ public class ProjectParameterServiceTest { .thenReturn(false); Result result = - projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null); + projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null, + DataType.VARCHAR.name()); assertNull(result.getData()); assertNull(result.getCode()); assertNull(result.getMsg()); @@ -235,9 +248,10 @@ public class ProjectParameterServiceTest { Page page = new Page<>(pageNo, pageSize); page.setRecords(Collections.singletonList(getProjectParameter())); - when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page); + when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any(), any())) + .thenReturn(page); result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, - null); + null, null); assertEquals(Status.SUCCESS.getCode(), result.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index da5ea88c83..bca8a69a16 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -92,24 +92,4 @@ public class CheckUtilsTest { Assertions.assertTrue(CheckUtils.checkPhone("17362537263")); } - /** - * check file path - */ - @Test - public void testCheckFilePath() { - // true - Assertions.assertTrue(CheckUtils.checkFilePath("/")); - Assertions.assertTrue(CheckUtils.checkFilePath("xx/")); - Assertions.assertTrue(CheckUtils.checkFilePath("/xx")); - Assertions.assertTrue(CheckUtils.checkFilePath("14567134578654")); - Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/")); - Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/1531531..13513/153135..")); - // false - Assertions.assertFalse(CheckUtils.checkFilePath(null)); - Assertions.assertFalse(CheckUtils.checkFilePath("file://xxx/ss")); - Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss;/dasd/123")); - Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss && /dasd/123")); - Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss || /dasd/123")); - } - } diff --git a/dolphinscheduler-api/src/test/resources/application.yaml b/dolphinscheduler-api/src/test/resources/application.yaml index 26536d631f..5eb7e1f8d7 100644 --- a/dolphinscheduler-api/src/test/resources/application.yaml +++ b/dolphinscheduler-api/src/test/resources/application.yaml @@ -44,6 +44,17 @@ mybatis-plus: registry: type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 600ms + digest: ~ api: audit-enable: true diff --git a/dolphinscheduler-api/src/test/resources/logback-spring.xml b/dolphinscheduler-api/src/test/resources/logback.xml similarity index 100% rename from dolphinscheduler-api/src/test/resources/logback-spring.xml rename to dolphinscheduler-api/src/test/resources/logback.xml diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml new file mode 100644 index 0000000000..9bad537cd9 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-authentication + dev-SNAPSHOT + + + dolphinscheduler-aws-authentication + + + + + com.amazonaws + aws-java-sdk-emr + + + + com.amazonaws + aws-java-sdk-s3 + + + + com.amazonaws + aws-java-sdk-sagemaker + + + + com.amazonaws + aws-java-sdk-dms + + + + software.amazon.awssdk + datasync + + + + org.slf4j + slf4j-api + provided + + + + + diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java new file mode 100644 index 0000000000..c79c70d8c2 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderFactor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import static org.apache.dolphinscheduler.authentication.aws.AwsConfigurationKeys.AWS_AUTHENTICATION_TYPE; + +import java.util.Map; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; + +@Slf4j +@UtilityClass +public class AWSCredentialsProviderFactor { + + public static AWSCredentialsProvider credentialsProvider(Map awsProperties) { + String awsAuthenticationType = awsProperties.getOrDefault( + AWS_AUTHENTICATION_TYPE, AWSCredentialsProviderType.STATIC_CREDENTIALS_PROVIDER.getName()); + AWSCredentialsProviderType awsCredentialsProviderType = + AWSCredentialsProviderType.of(awsAuthenticationType).orElse(null); + if (awsCredentialsProviderType == null) { + throw new IllegalArgumentException( + "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); + } + switch (awsCredentialsProviderType) { + case STATIC_CREDENTIALS_PROVIDER: + return createAWSStaticCredentialsProvider(awsProperties); + case INSTANCE_PROFILE_CREDENTIALS_PROVIDER: + return createInstanceProfileCredentialsProvider(); + default: + throw new IllegalArgumentException( + "The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); + } + + } + + private static AWSCredentialsProvider createAWSStaticCredentialsProvider(Map awsProperties) { + String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); + String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); + AWSStaticCredentialsProvider awsStaticCredentialsProvider = + new AWSStaticCredentialsProvider(basicAWSCredentials); + log.info("AWSStaticCredentialsProvider created successfully"); + return awsStaticCredentialsProvider; + } + + private static AWSCredentialsProvider createInstanceProfileCredentialsProvider() { + InstanceProfileCredentialsProvider instanceProfileCredentialsProvider = + InstanceProfileCredentialsProvider.getInstance(); + log.info("InstanceProfileCredentialsProvider created successfully"); + return instanceProfileCredentialsProvider; + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java new file mode 100644 index 0000000000..9b932f1554 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSCredentialsProviderType.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import java.util.Optional; + +import lombok.Getter; + +@Getter +public enum AWSCredentialsProviderType { + + STATIC_CREDENTIALS_PROVIDER("AWSStaticCredentialsProvider"), + INSTANCE_PROFILE_CREDENTIALS_PROVIDER("InstanceProfileCredentialsProvider"), + ; + + private final String name; + + AWSCredentialsProviderType(String name) { + this.name = name; + } + + public static Optional of(String name) { + if (name == null) { + return Optional.empty(); + } + for (AWSCredentialsProviderType type : values()) { + if (type.getName().equalsIgnoreCase(name)) { + return Optional.of(type); + } + } + return Optional.empty(); + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java new file mode 100644 index 0000000000..2056268f2b --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AWSDatabaseMigrationServiceClientFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import java.util.Map; + +import lombok.experimental.UtilityClass; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService; +import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder; + +@UtilityClass +public class AWSDatabaseMigrationServiceClientFactory { + + public AWSDatabaseMigrationService createAWSDatabaseMigrationServiceClient(Map awsProperties) { + AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); + Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); + String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); + + if (endpoint != null && !endpoint.isEmpty()) { + return AWSDatabaseMigrationServiceClientBuilder + .standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) + .withCredentials(awsCredentialsProvider) + .build(); + } else { + return AWSDatabaseMigrationServiceClientBuilder + .standard() + .withCredentials(awsCredentialsProvider) + .withRegion(regions) + .build(); + } + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java new file mode 100644 index 0000000000..ea00473b79 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonElasticMapReduceClientFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import java.util.Map; + +import lombok.experimental.UtilityClass; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; +import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; + +@UtilityClass +public class AmazonElasticMapReduceClientFactory { + + public AmazonElasticMapReduce createAmazonElasticMapReduceClient(Map awsProperties) { + AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); + Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); + String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); + + if (endpoint != null && !endpoint.isEmpty()) { + return AmazonElasticMapReduceClientBuilder + .standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) + .withCredentials(awsCredentialsProvider) + .build(); + } else { + return AmazonElasticMapReduceClientBuilder + .standard() + .withCredentials(awsCredentialsProvider) + .withRegion(regions) + .build(); + } + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java new file mode 100644 index 0000000000..c45e4de9ea --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonS3ClientFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import java.util.Map; + +import lombok.experimental.UtilityClass; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; + +@UtilityClass +public class AmazonS3ClientFactory { + + public AmazonS3 createAmazonS3Client(Map awsProperties) { + AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); + Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); + String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); + + if (endpoint != null && !endpoint.isEmpty()) { + return AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) + .withCredentials(awsCredentialsProvider) + .build(); + } else { + return AmazonS3ClientBuilder + .standard() + .withCredentials(awsCredentialsProvider) + .withRegion(regions) + .build(); + } + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java new file mode 100644 index 0000000000..6bff921894 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AmazonSageMakerClientFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import java.util.Map; + +import lombok.experimental.UtilityClass; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.sagemaker.AmazonSageMaker; +import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder; + +@UtilityClass +public class AmazonSageMakerClientFactory { + + public AmazonSageMaker createAmazonSageMakerClient(Map awsProperties) { + AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); + Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); + String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); + + if (endpoint != null && !endpoint.isEmpty()) { + return AmazonSageMakerClientBuilder + .standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) + .withCredentials(awsCredentialsProvider) + .build(); + } else { + return AmazonSageMakerClientBuilder + .standard() + .withCredentials(awsCredentialsProvider) + .withRegion(regions) + .build(); + } + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java new file mode 100644 index 0000000000..3d2d8677b3 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/AwsConfigurationKeys.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +public class AwsConfigurationKeys { + + public static final String AWS_AUTHENTICATION_TYPE = "credentials.provider.type"; + public static final String AWS_REGION = "region"; + public static final String AWS_ENDPOINT = "endpoint"; + + public static final String AWS_ACCESS_KEY_ID = "access.key.id"; + public static final String AWS_SECRET = "access.key.secret"; +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java new file mode 100644 index 0000000000..b67c7dd840 --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/java/org/apache/dolphinscheduler/authentication/aws/DataSyncClientFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.authentication.aws; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.datasync.DataSyncClient; + +import java.util.Map; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class DataSyncClientFactory { + + public DataSyncClient createDataSyncClient(Map awsProperties) { + // todo: upgrade the version of aws sdk + String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); + String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); + final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey); + final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials); + + // create a datasync client + return DataSyncClient.builder() + .region(Region.of(awsProperties.get(AwsConfigurationKeys.AWS_REGION))) + .credentialsProvider(awsCredentialsProvider) + .build(); + } + +} diff --git a/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml new file mode 100644 index 0000000000..6d453bb78a --- /dev/null +++ b/dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources/aws.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +aws: + s3: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: accessKey123 + access.key.secret: secretKey123 + region: us-east-1 + bucket.name: dolphinscheduler + endpoint: http://s3:9000 + emr: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + sagemaker: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + dms: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + datasync: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + diff --git a/dolphinscheduler-authentication/pom.xml b/dolphinscheduler-authentication/pom.xml new file mode 100644 index 0000000000..b49c4d37d0 --- /dev/null +++ b/dolphinscheduler-authentication/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + dev-SNAPSHOT + + + dolphinscheduler-authentication + pom + + + dolphinscheduler-aws-authentication + + + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + + diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 61cb58dbbc..0a7c7389de 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -37,6 +37,7 @@ 1.2.20 2.12.0 0.5.11 + 0.7.1 1.41.0 1.11 @@ -955,6 +956,13 @@ + + org.testcontainers + testcontainers + ${testcontainer.version} + test + + org.testcontainers mysql diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index e58519e854..cb4a2aea96 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -41,6 +41,10 @@ + + org.apache.dolphinscheduler + dolphinscheduler-aws-authentication + commons-io commons-io diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index cc07accc9b..5ec2a2b181 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -36,6 +36,7 @@ public final class Constants { public static final String COMMON_PROPERTIES_PATH = "/common.properties"; public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; + public static final String AWS_YAML_PATH = "/aws.yaml"; public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; @@ -130,8 +131,7 @@ public final class Constants { */ public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; - public static final String AWS_S3_BUCKET_NAME = "resource.aws.s3.bucket.name"; - public static final String AWS_END_POINT = "resource.aws.s3.endpoint"; + public static final String AWS_S3_BUCKET_NAME = "aws.s3.bucket.name"; public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name"; public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint"; @@ -249,11 +249,6 @@ public final class Constants { */ public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,39}$"); - /** - * file path regex - */ - public static final Pattern REGEX_FILE_PATH = Pattern.compile("^[a-zA-Z0-9_./-]+$"); - /** * read permission */ @@ -704,19 +699,8 @@ public final class Constants { public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint"; - /** - * remote logging for S3 - */ - public static final String REMOTE_LOGGING_S3_ACCESS_KEY_ID = "remote.logging.s3.access.key.id"; - - public static final String REMOTE_LOGGING_S3_ACCESS_KEY_SECRET = "remote.logging.s3.access.key.secret"; - public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name"; - public static final String REMOTE_LOGGING_S3_ENDPOINT = "remote.logging.s3.endpoint"; - - public static final String REMOTE_LOGGING_S3_REGION = "remote.logging.s3.region"; - /** * remote logging for GCS */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/TenantConstants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/TenantConstants.java index 8bffd5429e..aa905cbb72 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/TenantConstants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/TenantConstants.java @@ -21,5 +21,5 @@ public class TenantConstants { public static final String DEFAULT_TENANT_CODE = "default"; - public static final String BOOTSTRAPT_SYSTEM_USER = System.getProperty("user.name"); + public static final String BOOTSTRAP_SYSTEM_USER = System.getProperty("user.name"); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java index 1e4f49721a..afa7e97023 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ServerStatus.java @@ -20,6 +20,6 @@ package org.apache.dolphinscheduler.common.enums; public enum ServerStatus { NORMAL, - BUSY + BUSY, } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java index 54dba2d5bd..4fef7b032a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.log.remote; +import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -26,41 +27,25 @@ import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Map; import lombok.extern.slf4j.Slf4j; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; @Slf4j public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { - private String accessKeyId; + private final String bucketName; - private String accessKeySecret; - - private String region; - - private String bucketName; - - private String endPoint; - - private AmazonS3 s3Client; + private final AmazonS3 s3Client; private static S3RemoteLogHandler instance; private S3RemoteLogHandler() { - accessKeyId = readAccessKeyID(); - accessKeySecret = readAccessKeySecret(); - region = readRegion(); bucketName = readBucketName(); - endPoint = readEndPoint(); s3Client = buildS3Client(); checkBucketNameExists(bucketName); } @@ -74,23 +59,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { } protected AmazonS3 buildS3Client() { - if (StringUtils.isNotEmpty(endPoint)) { - return AmazonS3ClientBuilder - .standard() - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( - endPoint, Regions.fromName(region).getName())) - .withCredentials( - new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) - .build(); - } else { - return AmazonS3ClientBuilder - .standard() - .withCredentials( - new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) - .withRegion(Regions.fromName(region)) - .build(); - } + Map awsProperties = PropertyUtils.getByPrefix("aws.s3.", ""); + return AmazonS3ClientFactory.createAmazonS3Client(awsProperties); } @Override @@ -131,24 +101,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { } } - protected String readAccessKeyID() { - return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_ID); - } - - protected String readAccessKeySecret() { - return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_SECRET); - } - - protected String readRegion() { - return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_REGION); - } - protected String readBucketName() { - return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_BUCKET_NAME); - } - - protected String readEndPoint() { - return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ENDPOINT); + return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME); } public void checkBucketNameExists(String bucketName) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java index 9faaef82be..8533ca6e48 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/AlertServerHeartBeat.java @@ -24,4 +24,9 @@ import lombok.experimental.SuperBuilder; @NoArgsConstructor public class AlertServerHeartBeat extends BaseHeartBeat implements HeartBeat { + /** + * If the alert server is active or standby + */ + private boolean isActive; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java new file mode 100644 index 0000000000..bf127d14bb --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.dolphinscheduler.common.utils; + +import lombok.extern.slf4j.Slf4j; + +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.constructor.Constructor; + +/** + * Whitelist constructor implementation for YAML snake. + * Copied from Apache ShardingSphere and Apache Skywalking. + */ +@Slf4j +public final class ClassFilterConstructor extends Constructor { + + private final Class[] acceptClasses; + + public ClassFilterConstructor(final Class[] acceptClasses) { + super(new LoaderOptions()); + this.acceptClasses = acceptClasses; + } + + @Override + protected Class getClassForName(final String name) throws ClassNotFoundException { + for (Class each : acceptClasses) { + if (name.equals(each.getName())) { + log.info("name - {} : class - {}", name, super.getClassForName(name)); + return super.getClassForName(name); + } + } + throw new IllegalArgumentException(String.format("Class is not accepted: %s", name)); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 82d4de9599..aee1589335 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; @@ -42,7 +43,7 @@ public class PropertyUtils { private final ImmutablePriorityPropertyDelegate propertyDelegate = new ImmutablePriorityPropertyDelegate( new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), - new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); @@ -106,6 +107,19 @@ public class PropertyUtils { return matchedProperties; } + /** + * Get all properties with specified prefix, like: fs., will replace the prefix with newPrefix + */ + public static Map getByPrefix(String prefix, String newPrefix) { + Map matchedProperties = new HashMap<>(); + for (String propName : propertyDelegate.getPropertyKeys()) { + if (propName.startsWith(prefix)) { + matchedProperties.put(propName.replace(prefix, newPrefix), propertyDelegate.get(propName)); + } + } + return matchedProperties; + } + public static Set getSet(String key, Function> transformFunction, Set defaultValue) { return propertyDelegate.get(key, transformFunction, defaultValue); } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index fdb553b4bc..cf1723700e 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -39,17 +39,6 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id=minioadmin -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=minioadmin -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=cn-north-1 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint=http://localhost:9000 - # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS @@ -188,16 +177,7 @@ remote.logging.oss.access.key.secret= remote.logging.oss.bucket.name= # oss endpoint, required if you set remote.logging.target=OSS remote.logging.oss.endpoint= -# s3 access key id, required if you set remote.logging.target=S3 -remote.logging.s3.access.key.id= -# s3 access key secret, required if you set remote.logging.target=S3 -remote.logging.s3.access.key.secret= -# s3 bucket name, required if you set remote.logging.target=S3 -remote.logging.s3.bucket.name= -# s3 endpoint, required if you set remote.logging.target=S3 -remote.logging.s3.endpoint= -# s3 region, required if you set remote.logging.target=S3 -remote.logging.s3.region= + # the location of the google cloud credential, required if you set remote.logging.target=GCS remote.logging.google.cloud.storage.credential=/path/to/credential # gcs bucket name, required if you set remote.logging.target=GCS diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml index 2cb48750a4..f413958e64 100644 --- a/dolphinscheduler-common/src/main/resources/remote-logging.yaml +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -34,28 +34,16 @@ remote-logging: bucket.name: # oss endpoint, required if you set remote-logging.target=OSS endpoint: - # required if you set remote-logging.target=S3 - s3: - # s3 access key id, required if you set remote-logging.target=S3 - access.key.id: - # s3 access key secret, required if you set remote-logging.target=S3 - access.key.secret: - # s3 bucket name, required if you set remote-logging.target=S3 - bucket.name: - # s3 endpoint, required if you set remote-logging.target=S3 - endpoint: - # s3 region, required if you set remote-logging.target=S3 - region: google.cloud.storage: # the location of the google cloud credential, required if you set remote-logging.target=GCS credential: /path/to/credential - # gcs bucket name, required if you set remote-logging.target=GCS + # gcs bucket name, required if you set remote-logging.target=GCS bucket.name: abs: # abs account name, required if you set resource.storage.type=ABS account.name: - # abs account key, required if you set resource.storage.type=ABS + # abs account key, required if you set resource.storage.type=ABS account.key: - # abs container name, required if you set resource.storage.type=ABS + # abs container name, required if you set resource.storage.type=ABS container.name: diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index c915197844..8abfb38269 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -17,12 +17,15 @@ package org.apache.dolphinscheduler.common.utils; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Collections; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -48,4 +51,14 @@ public class PropertyUtilsTest { }, Sets.newHashSet("docker0")); Assertions.assertEquals(Sets.newHashSet("docker0"), networkInterface); } + + @Test + void getByPrefix() { + Map awsProperties = PropertyUtils.getByPrefix("resource.aws.", ""); + assertThat(awsProperties).containsEntry("access.key.id", "minioadmin"); + assertThat(awsProperties).containsEntry("secret.access.key", "minioadmin"); + assertThat(awsProperties).containsEntry("region", "cn-north-1"); + assertThat(awsProperties).containsEntry("s3.bucket.name", "dolphinscheduler"); + assertThat(awsProperties).containsEntry("endpoint", "http://localhost:9000"); + } } diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties index 7f66a32a23..ce8ef3bf4f 100644 --- a/dolphinscheduler-common/src/test/resources/common.properties +++ b/dolphinscheduler-common/src/test/resources/common.properties @@ -45,6 +45,10 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 +# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider +# AWSStaticCredentialsProvider: use the access key and secret key to authenticate +# InstanceProfileCredentialsProvider: use the IAM role to authenticate +aws.credentials.provider.type=AWSStaticCredentialsProvider # The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required resource.aws.access.key.id=minioadmin # The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required @@ -54,7 +58,7 @@ resource.aws.region=cn-north-1 # The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. resource.aws.s3.bucket.name=dolphinscheduler # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint=http://localhost:9000 +resource.aws.endpoint=http://localhost:9000 # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= diff --git a/dolphinscheduler-common/src/test/resources/logback.xml b/dolphinscheduler-common/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-common/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 3b71312d0f..d9f6109834 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -63,8 +63,7 @@ import com.google.common.collect.Lists; @Slf4j public class AlertDao { - @Value("${alert.query_alert_threshold:100}") - private Integer QUERY_ALERT_THRESHOLD; + private static final Integer QUERY_ALERT_THRESHOLD = 100; @Value("${alert.alarm-suppression.crash:60}") private Integer crashAlarmSuppression; @@ -104,8 +103,8 @@ public class AlertDao { * update alert sending(execution) status * * @param alertStatus alertStatus - * @param log alert results json - * @param id id + * @param log alert results json + * @param id id * @return update alert result */ public int updateAlert(AlertStatus alertStatus, String log, int id) { @@ -134,9 +133,9 @@ public class AlertDao { /** * add AlertSendStatus * - * @param sendStatus alert send status - * @param log log - * @param alertId alert id + * @param sendStatus alert send status + * @param log log + * @param alertId alert id * @param alertPluginInstanceId alert plugin instance id * @return insert count */ @@ -192,7 +191,7 @@ public class AlertDao { * process time out alert * * @param processInstance processInstance - * @param projectUser projectUser + * @param projectUser projectUser */ public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProjectUser projectUser) { int alertGroupId = processInstance.getWarningGroupId(); @@ -238,8 +237,8 @@ public class AlertDao { * task timeout warn * * @param processInstance processInstanceId - * @param taskInstance taskInstance - * @param projectUser projectUser + * @param taskInstance taskInstance + * @param projectUser projectUser */ public void sendTaskTimeoutAlert(ProcessInstance processInstance, TaskInstance taskInstance, ProjectUser projectUser) { @@ -271,10 +270,11 @@ public class AlertDao { } /** - * List alerts that are pending for execution + * List pending alerts which id > minAlertId and status = {@link AlertStatus#WAIT_EXECUTION} order by id asc. */ - public List listPendingAlerts() { - return alertMapper.listingAlertByStatus(AlertStatus.WAIT_EXECUTION.getCode(), QUERY_ALERT_THRESHOLD); + public List listPendingAlerts(int minAlertId) { + return alertMapper.listingAlertByStatus(minAlertId, AlertStatus.WAIT_EXECUTION.getCode(), + QUERY_ALERT_THRESHOLD); } public List listAlerts(int processInstanceId) { @@ -283,15 +283,6 @@ public class AlertDao { return alertMapper.selectList(wrapper); } - /** - * for test - * - * @return AlertMapper - */ - public AlertMapper getAlertMapper() { - return alertMapper; - } - /** * list all alert plugin instance by alert group id * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java index 03e9140145..4343d07c29 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectParameter.java @@ -55,6 +55,9 @@ public class ProjectParameter { @TableField("param_value") private String paramValue; + @TableField("param_data_type") + private String paramDataType; + private Date createTime; private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java index c30c1c9043..aab7b6f5f2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java @@ -34,9 +34,10 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; public interface AlertMapper extends BaseMapper { /** - * Query the alert by alertStatus and return limit with default sort. + * Query the alert which id > minAlertId and status = alertStatus order by id asc. */ - List listingAlertByStatus(@Param("alertStatus") int alertStatus, @Param("limit") int limit); + List listingAlertByStatus(@Param("minAlertId") int minAlertId, @Param("alertStatus") int alertStatus, + @Param("limit") int limit); /** * Insert server crash alert diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index 8c8314e7cc..ff0f6243ce 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -26,6 +26,8 @@ import java.util.Date; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * command mapper interface @@ -50,7 +52,7 @@ public interface CommandMapper extends BaseMapper { * * @return */ - List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); + IPage queryCommandPage(Page page); List queryCommandByIdSlot(@Param("currentSlotIndex") int currentSlotIndex, @Param("totalSlot") int totalSlot, @@ -58,4 +60,7 @@ public interface CommandMapper extends BaseMapper { @Param("fetchNumber") int fetchNum); void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); + + IPage queryCommandPageByIds(Page page, + @Param("workflowDefinitionCodes") List workflowDefinitionCodes); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java index 8054a91786..9dee5bbc32 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java @@ -26,6 +26,8 @@ import java.util.Date; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * error command mapper interface @@ -43,4 +45,9 @@ public interface ErrorCommandMapper extends BaseMapper { @Param("startTime") Date startTime, @Param("endTime") Date endTime, @Param("projectCodes") List projectCodes); + + IPage queryErrorCommandPage(Page page); + + IPage queryErrorCommandPageByIds(Page page, + @Param("workflowDefinitionCodes") List workflowDefinitionCodes); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java index 820ac3b3a6..f3e187f803 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.java @@ -34,7 +34,8 @@ public interface ListenerEventMapper extends BaseMapper { void insertServerDownEvent(@Param("event") ListenerEvent event, @Param("crashAlarmSuppressionStartTime") Date crashAlarmSuppressionStartTime); - List listingListenerEventByStatus(@Param("postStatus") AlertStatus postStatus, + List listingListenerEventByStatus(@Param("minId") int minId, + @Param("postStatus") int postStatus, @Param("limit") int limit); void updateListenerEvent(@Param("eventId") int eventId, @Param("postStatus") AlertStatus postStatus, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java index 67140720fe..67522caf50 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java @@ -188,4 +188,5 @@ public interface ProcessDefinitionMapper extends BaseMapper { * @return project ids list */ List listProjectIds(); + List queryDefinitionCodeListByProjectCodes(@Param("projectCodes") List projectCodes); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java index 0e7810158c..3f700a9a84 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.java @@ -38,7 +38,8 @@ public interface ProjectParameterMapper extends BaseMapper { IPage queryProjectParameterListPaging(IPage page, @Param("projectCode") long projectCode, @Param("projectParameterIds") List projectParameterIds, - @Param("searchName") String searchName); + @Param("searchName") String searchName, + @Param("projectParameterDataType") String projectParameterDataType); List queryByProjectCode(@Param("projectCode") long projectCode); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java new file mode 100644 index 0000000000..424c616cd3 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/ListenerEventDao.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; + +import java.util.Date; +import java.util.List; + +public interface ListenerEventDao extends IDao { + + List listingPendingEvents(int minId, int limit); + + void updateListenerEvent(int eventId, AlertStatus alertStatus, String message, Date date); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java new file mode 100644 index 0000000000..06c4dccd5e --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImpl.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.mapper.ListenerEventMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.Date; +import java.util.List; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Repository; + +@Slf4j +@Repository +public class ListenerEventDaoImpl extends BaseDao implements ListenerEventDao { + + public ListenerEventDaoImpl(@NonNull ListenerEventMapper listenerEventMapper) { + super(listenerEventMapper); + } + + @Override + public List listingPendingEvents(int minId, int limit) { + return mybatisMapper.listingListenerEventByStatus(minId, AlertStatus.WAIT_EXECUTION.getCode(), limit); + } + + @Override + public void updateListenerEvent(int eventId, AlertStatus alertStatus, String message, Date date) { + mybatisMapper.updateListenerEvent(eventId, alertStatus, message, date); + } +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index e56afa830e..f0c32aae78 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -55,7 +55,9 @@ select from t_ds_alert - where alert_status = #{alertStatus} + where id > #{minAlertId} + and alert_status = #{alertStatus} + order by id asc limit #{limit} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index 16f7c05f25..7f587ef899 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -34,10 +34,22 @@ group by cmd.command_type + + + + + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml index ae76bfcadd..8e6d3bbc3a 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapper.xml @@ -60,7 +60,8 @@ select from t_ds_listener_event - where post_status = #{postStatus.code} + where id > #{minId} and post_status = #{postStatus} + order by id asc limit #{limit} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index 77775b7668..79f0d54931 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -208,4 +208,14 @@ SELECT DISTINCT(id) as project_id FROM t_ds_project + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml index 5b22d40a81..4e1e972a04 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapper.xml @@ -19,7 +19,7 @@ - id, param_name, param_value, code, project_code, user_id, operator, create_time, update_time + id, param_name, param_value, param_data_type, code, project_code, user_id, operator, create_time, update_time select - pp.id, param_name, param_value, code, project_code, user_id, operator, pp.create_time, pp.update_time, + pp.id, param_name, param_value, param_data_type, code, project_code, user_id, operator, pp.create_time, pp.update_time, u.user_name as create_user, u2.user_name as modify_user from t_ds_project_parameter pp @@ -69,6 +69,9 @@ OR param_value LIKE concat('%', #{searchName}, '%') ) + + AND param_data_type = #{projectParameterDataType} + order by pp.update_time desc diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 42c893bb05..a907ec69a1 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -667,6 +667,7 @@ CREATE TABLE t_ds_project_parameter id int(11) NOT NULL AUTO_INCREMENT, param_name varchar(255) NOT NULL, param_value text NOT NULL, + param_data_type varchar(50) DEFAULT 'VARCHAR', code bigint(20) NOT NULL, project_code bigint(20) NOT NULL, user_id int(11) DEFAULT NULL, diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index b30bc13887..3a4f17a357 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -668,6 +668,7 @@ CREATE TABLE `t_ds_project_parameter` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', `param_name` varchar(255) NOT NULL COMMENT 'project parameter name', `param_value` text NOT NULL COMMENT 'project parameter value', + `param_data_type` varchar(50) DEFAULT 'VARCHAR' COMMENT 'project parameter data type', `code` bigint(20) NOT NULL COMMENT 'encoding', `project_code` bigint(20) NOT NULL COMMENT 'project code', `user_id` int(11) DEFAULT NULL COMMENT 'creator id', diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 2d224092c4..d314c6ef09 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -595,6 +595,7 @@ CREATE TABLE t_ds_project_parameter ( id int NOT NULL , param_name varchar(255) NOT NULL , param_value text NOT NULL , + param_data_type varchar(50) DEFAULT 'VARCHAR', code bigint NOT NULL, project_code bigint NOT NULL, user_id int DEFAULT NULL , diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_ddl.sql index 393bda87fc..fbc9b26b86 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -40,7 +40,8 @@ ALTER TABLE t_ds_alert ADD COLUMN IF NOT EXISTS "process_instance_id" int DEFAUL ALTER TABLE t_ds_alert ADD COLUMN IF NOT EXISTS "alert_type" int DEFAULT NULL; --- Add unique key -CREATE UNIQUE INDEX IF NOT EXISTS t_ds_relation_project_user_un on t_ds_relation_project_user (user_id, project_id); + +CREATE INDEX IF NOT EXISTS t_ds_relation_project_user_un on t_ds_relation_project_user (user_id, project_id); CREATE UNIQUE INDEX IF NOT EXISTS unique_name on t_ds_project (name); CREATE UNIQUE INDEX IF NOT EXISTS unique_code on t_ds_project (code); CREATE UNIQUE INDEX IF NOT EXISTS unique_queue_name on t_ds_queue (queue_name); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql index d10ac6b710..000173b397 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -53,4 +53,6 @@ END; d// delimiter ; CALL modify_data_t_ds_audit_log_input_entry; -DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; \ No newline at end of file +DROP PROCEDURE modify_data_t_ds_audit_log_input_entry; + +ALTER TABLE t_ds_project_parameter ADD `param_data_type` varchar (50) DEFAULT 'VARCHAR' COMMENT 'project parameter data type'; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql index ee06588e58..4b4cb66651 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -56,4 +56,6 @@ $$ LANGUAGE plpgsql; d// select modify_data_t_ds_audit_log_input_entry(); -DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); \ No newline at end of file +DROP FUNCTION IF EXISTS modify_data_t_ds_audit_log_input_entry(); + +ALTER TABLE t_ds_project_parameter ADD COLUMN IF NOT EXISTS param_data_type varchar(50) DEFAULT 'VARCHAR'; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 560b68754a..2105885fa3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -42,6 +42,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.collect.Lists; /** @@ -64,6 +67,18 @@ public class CommandMapperTest extends BaseDaoTest { Assertions.assertTrue(command.getId() > 0); } + @Test + public void testQueryCommandPageByIds() { + Command expectedCommand = createCommand(); + Page page = new Page<>(1, 10); + IPage commandIPage = commandMapper.queryCommandPageByIds(page, + Lists.newArrayList(expectedCommand.getProcessDefinitionCode())); + List commandList = commandIPage.getRecords(); + assertThat(commandList).isNotEmpty(); + assertThat(commandIPage.getTotal()).isEqualTo(1); + assertThat(commandList.get(0).getId()).isEqualTo(expectedCommand.getId()); + } + /** * test select by id */ @@ -135,7 +150,7 @@ public class CommandMapperTest extends BaseDaoTest { createCommand(CommandType.START_PROCESS, processDefinition.getCode()); - List actualCommand = commandMapper.queryCommandPage(1, 0); + List actualCommand = commandMapper.selectList(new QueryWrapper<>()); Assertions.assertNotNull(actualCommand); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 098c001f2d..07409cf0e5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.dao.mapper; +import static com.google.common.truth.Truth.assertThat; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.CommandCount; @@ -31,6 +33,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + public class ErrorCommandMapperTest extends BaseDaoTest { @Autowired @@ -54,6 +59,18 @@ public class ErrorCommandMapperTest extends BaseDaoTest { return errorCommand; } + @Test + public void testQueryCommandPageByIds() { + ErrorCommand expectedCommand = insertOne(); + Page page = new Page<>(1, 10); + IPage commandIPage = errorCommandMapper.queryErrorCommandPageByIds(page, + Lists.newArrayList(expectedCommand.getProcessDefinitionCode())); + List commandList = commandIPage.getRecords(); + assertThat(commandList).isNotEmpty(); + assertThat(commandIPage.getTotal()).isEqualTo(1); + assertThat(commandList.get(0).getId()).isEqualTo(expectedCommand.getId()); + } + /** * test query */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java index f3e877aaa3..4e239265c4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ListenerEventMapperTest.java @@ -81,7 +81,7 @@ public class ListenerEventMapperTest extends BaseDaoTest { ListenerEvent event2 = generateServerDownListenerEvent("192.168.x.2"); listenerEventMapper.batchInsert(Lists.newArrayList(event1, event2)); List listenerEvents = - listenerEventMapper.listingListenerEventByStatus(AlertStatus.WAIT_EXECUTION, 50); + listenerEventMapper.listingListenerEventByStatus(-1, AlertStatus.WAIT_EXECUTION.getCode(), 50); Assertions.assertEquals(listenerEvents.size(), 2); } @@ -111,8 +111,10 @@ public class ListenerEventMapperTest extends BaseDaoTest { ListenerEvent actualAlert = listenerEventMapper.selectById(event.getId()); Assertions.assertNull(actualAlert); } + /** * create server down event + * * @param host worker host * @return listener event */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java index 48e51d9ca3..1c13d95a0e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectParameterMapperTest.java @@ -89,7 +89,7 @@ public class ProjectParameterMapperTest extends BaseDaoTest { insertOne(2, "name2", 2); Page page = new Page(1, 3); - IPage res = projectParameterMapper.queryProjectParameterListPaging(page, 1, null, null); + IPage res = projectParameterMapper.queryProjectParameterListPaging(page, 1, null, null, null); Assertions.assertEquals(1, res.getRecords().size()); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index fe8545f3e2..a2c2c2ab98 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -18,37 +18,23 @@ package org.apache.dolphinscheduler.dao.repository.impl; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.ProfileType; import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.Alert; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.transaction.annotation.Transactional; -@ActiveProfiles(ProfileType.H2) -@ExtendWith(MockitoExtension.class) -@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) -@SpringBootTest(classes = DaoConfiguration.class) -@Transactional -@Rollback -public class AlertDaoTest { +class AlertDaoTest extends BaseDaoTest { @Autowired private AlertDao alertDao; @Test - public void testAlertDao() { + void testAlertDao() { Alert alert = new Alert(); alert.setTitle("Mysql Exception"); alert.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " @@ -57,25 +43,25 @@ public class AlertDaoTest { alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); alertDao.addAlert(alert); - List alerts = alertDao.listPendingAlerts(); + List alerts = alertDao.listPendingAlerts(-1); Assertions.assertNotNull(alerts); Assertions.assertNotEquals(0, alerts.size()); } @Test - public void testAddAlertSendStatus() { + void testAddAlertSendStatus() { int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS, "success", 1, 1); Assertions.assertEquals(1, insertCount); } @Test - public void testSendServerStoppedAlert() { + void testSendServerStoppedAlert() { int alertGroupId = 1; String host = "127.0.0.998165432"; String serverType = "Master"; alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); - long count = alertDao.listPendingAlerts() + long count = alertDao.listPendingAlerts(-1) .stream() .filter(alert -> alert.getContent().contains(host)) .count(); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java new file mode 100644 index 0000000000..2574c52f78 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ListenerEventDaoImplTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.ListenerEventType; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.ListenerEvent; +import org.apache.dolphinscheduler.dao.repository.ListenerEventDao; + +import java.util.Date; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class ListenerEventDaoImplTest extends BaseDaoTest { + + @Autowired + private ListenerEventDao listenerEventDao; + + @Test + void listingPendingEvents() { + int minId = -1; + int limit = 10; + assertThat(listenerEventDao.listingPendingEvents(minId, limit)).isEmpty(); + + ListenerEvent listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.WAIT_EXECUTION) + .build(); + listenerEventDao.insert(listenerEvent); + + listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.EXECUTION_SUCCESS) + .build(); + listenerEventDao.insert(listenerEvent); + + assertThat(listenerEventDao.listingPendingEvents(minId, limit)).hasSize(1); + } + + @Test + void updateListenerEvent() { + ListenerEvent listenerEvent = ListenerEvent.builder() + .eventType(ListenerEventType.SERVER_DOWN) + .sign("test") + .createTime(new Date()) + .updateTime(new Date()) + .postStatus(AlertStatus.WAIT_EXECUTION) + .build(); + listenerEventDao.insert(listenerEvent); + listenerEventDao.updateListenerEvent(listenerEvent.getId(), AlertStatus.EXECUTION_SUCCESS, "test", new Date()); + assertThat(listenerEventDao.queryById(listenerEvent.getId()).getPostStatus()) + .isEqualTo(AlertStatus.EXECUTION_SUCCESS); + } +} diff --git a/dolphinscheduler-data-quality/src/test/resources/logback.xml b/dolphinscheduler-data-quality/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-data-quality/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java index 751ac1ba08..973421615f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java @@ -37,6 +37,10 @@ public class DataSourceProcessorProvider { private DataSourceProcessorProvider() { } + public static void initialize() { + log.info("Initialize DataSourceProcessorProvider"); + } + public static DataSourceProcessor getDataSourceProcessor(@NonNull DbType dbType) { return dataSourcePluginManager.getDataSourceProcessorMap().get(dbType.name()); } diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 93962afd57..66c0eb7c09 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -525,7 +525,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. reactor-netty-core 1.0.22: https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.0.22, Apache 2.0 reactor-netty-http 1.0.22: https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-http/1.0.22, Apache 2.0 DmJdbcDriver18 8.1.2.79: https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18/8.1.2.79, Apache 2.0 - nimbus-jose-jwt 9.22: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.22, Apache 2.0 woodstox-core 6.4.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/6.4.0, Apache 2.0 auto-value 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value/1.10.1, Apache 2.0 auto-value-annotations 1.10.4: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.4, Apache 2.0 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml new file mode 100644 index 0000000000..6d453bb78a --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/aws.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +aws: + s3: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: accessKey123 + access.key.secret: secretKey123 + region: us-east-1 + bucket.name: dolphinscheduler + endpoint: http://s3:9000 + emr: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + sagemaker: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + dms: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + datasync: + # The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider + # AWSStaticCredentialsProvider: use the access key and secret key to authenticate + # InstanceProfileCredentialsProvider: use the IAM role to authenticate + credentials.provider.type: AWSStaticCredentialsProvider + access.key.id: minioadmin + access.key.secret: minioadmin + region: cn-north-1 + endpoint: http://localhost:9000 + diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties index 7583b3293a..604befdbf8 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties @@ -37,17 +37,6 @@ resource.azure.tenant.id=minioadmin # The query interval resource.query.interval=10000 -# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.access.key.id=accessKey123 -# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.secret.access.key=secretKey123 -# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required -resource.aws.region=us-east-1 -# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. -resource.aws.s3.bucket.name=dolphinscheduler -# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn -resource.aws.s3.endpoint=http://s3:9000 - # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id= # alibaba cloud access key secret, required if you set resource.storage.type=OSS diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml index 9a46ed02ad..ccfe940a6d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -34,6 +34,7 @@ services: retries: 120 volumes: - ./common.properties:/opt/dolphinscheduler/conf/common.properties + - ./aws.yaml:/opt/dolphinscheduler/conf/aws.yaml depends_on: s3: condition: service_healthy diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java index 832f3e1fab..e1db2a233e 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java @@ -37,6 +37,14 @@ public class AlertSendResponse { private List resResults; + public static AlertSendResponse success(List resResults) { + return new AlertSendResponse(true, resResults); + } + + public static AlertSendResponse fail(List resResults) { + return new AlertSendResponse(false, resResults); + } + @Data @NoArgsConstructor @AllArgsConstructor @@ -46,6 +54,14 @@ public class AlertSendResponse { private String message; + public static AlertSendResponseResult success() { + return new AlertSendResponseResult(true, null); + } + + public static AlertSendResponseResult fail(String message) { + return new AlertSendResponseResult(false, message); + } + } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java similarity index 83% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java index 519dd87199..b753f1efa7 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/AbstractClientMethodInvoker.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Method; -public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker { +abstract class AbstractClientMethodInvoker implements ClientMethodInvoker { protected final String methodIdentifier; @@ -32,7 +31,7 @@ public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker { protected final Host serverHost; - public BaseRemoteMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + AbstractClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { this.serverHost = serverHost; this.localMethod = localMethod; this.nettyRemotingClient = nettyRemotingClient; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java index d5c9ab73d3..41ec3e056d 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.extract.base.client; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.utils.Host; @@ -31,7 +30,7 @@ import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; @Slf4j -public class ClientInvocationHandler implements InvocationHandler { +class ClientInvocationHandler implements InvocationHandler { private final NettyRemotingClient nettyRemotingClient; @@ -39,7 +38,7 @@ public class ClientInvocationHandler implements InvocationHandler { private final Host serverHost; - public ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) { + ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) { this.serverHost = checkNotNull(serverHost); this.nettyRemotingClient = checkNotNull(nettyRemotingClient); this.methodInvokerMap = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java index dcf53b0311..a287fd95ce 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.extract.base.client; import java.lang.reflect.Method; -public interface ClientMethodInvoker { +interface ClientMethodInvoker { Object invoke(Object proxy, Method method, Object[] args) throws Throwable; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java index e60b0f18b0..afd3adf348 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.extract.base.client; -public interface IRpcClientProxyFactory { +interface IRpcClientProxyFactory { /** * Create the client proxy. diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java index 5635a88f34..bf329ab3fc 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Proxy; @@ -34,7 +33,7 @@ import com.google.common.cache.LoadingCache; /** * This class is used to create a proxy client which will transform local method invocation to remove invocation. */ -public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { +class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { private final NettyRemotingClient nettyRemotingClient; @@ -49,7 +48,7 @@ public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { } }); - public JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) { + JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) { this.nettyRemotingClient = nettyRemotingClient; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java similarity index 87% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java index b0d998af83..be570eb577 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java @@ -15,16 +15,15 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; +import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; import org.apache.dolphinscheduler.extract.base.protocal.Transporter; import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; -import java.util.concurrent.ExecutorService; - import lombok.extern.slf4j.Slf4j; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; @@ -38,11 +37,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private final NettyRemotingClient nettyRemotingClient; - private final ExecutorService callbackExecutor; - - public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) { + public NettyClientHandler(NettyRemotingClient nettyRemotingClient) { this.nettyRemotingClient = nettyRemotingClient; - this.callbackExecutor = callbackExecutor; } @Override @@ -64,13 +60,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { } StandardRpcResponse deserialize = JsonSerializer.deserialize(transporter.getBody(), StandardRpcResponse.class); future.setIRpcResponse(deserialize); - future.release(); - if (future.getInvokeCallback() != null) { - future.removeFuture(); - this.callbackExecutor.execute(future::executeInvokeCallback); - } else { - future.putResponse(deserialize); - } + future.putResponse(deserialize); } @Override diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java similarity index 56% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index e4682f5224..8ded68669d 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -15,35 +15,28 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; -import org.apache.dolphinscheduler.extract.base.exception.RemotingTooMuchRequestException; -import org.apache.dolphinscheduler.extract.base.future.InvokeCallback; -import org.apache.dolphinscheduler.extract.base.future.ReleaseSemaphore; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; import org.apache.dolphinscheduler.extract.base.protocal.Transporter; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; -import org.apache.dolphinscheduler.extract.base.utils.CallerThreadExecutePolicy; import org.apache.dolphinscheduler.extract.base.utils.Constants; import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; import java.net.InetSocketAddress; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.Bootstrap; @@ -63,7 +56,8 @@ public class NettyRemotingClient implements AutoCloseable { private final Bootstrap bootstrap = new Bootstrap(); - private final ConcurrentHashMap channels = new ConcurrentHashMap<>(128); + private final ReentrantLock channelsLock = new ReentrantLock(); + private final Map channels = new ConcurrentHashMap<>(); private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -71,14 +65,8 @@ public class NettyRemotingClient implements AutoCloseable { private final NettyClientConfig clientConfig; - private final Semaphore asyncSemaphore = new Semaphore(1024, true); - - private final ExecutorService callbackExecutor; - private final NettyClientHandler clientHandler; - private final ScheduledExecutorService responseFutureExecutor; - public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-"); @@ -87,18 +75,7 @@ public class NettyRemotingClient implements AutoCloseable { } else { this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory); } - this.callbackExecutor = new ThreadPoolExecutor( - Constants.CPUS, - Constants.CPUS, - 1, - TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), - ThreadUtils.newDaemonThreadFactory("NettyClientCallbackThread-"), - new CallerThreadExecutePolicy()); - this.clientHandler = new NettyClientHandler(this, callbackExecutor); - - this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor( - ThreadUtils.newDaemonThreadFactory("NettyClientResponseFutureThread-")); + this.clientHandler = new NettyClientHandler(this); this.start(); } @@ -127,74 +104,18 @@ public class NettyRemotingClient implements AutoCloseable { .addLast(new TransporterDecoder(), clientHandler, new TransporterEncoder()); } }); - this.responseFutureExecutor.scheduleWithFixedDelay(ResponseFuture::scanFutureTable, 0, 1, TimeUnit.SECONDS); isStarted.compareAndSet(false, true); } - public void sendAsync(final Host host, - final Transporter transporter, - final long timeoutMillis, - final InvokeCallback invokeCallback) throws InterruptedException, RemotingException { - final Channel channel = getChannel(host); - if (channel == null) { - throw new RemotingException("network error"); - } - /* - * request unique identification - */ - final long opaque = transporter.getHeader().getOpaque(); - /* - * control concurrency number - */ - boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); - if (acquired) { - final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore); - - /* - * response future - */ - final ResponseFuture responseFuture = new ResponseFuture(opaque, - timeoutMillis, - invokeCallback, - releaseSemaphore); - try { - channel.writeAndFlush(transporter).addListener(future -> { - if (future.isSuccess()) { - responseFuture.setSendOk(true); - return; - } else { - responseFuture.setSendOk(false); - } - responseFuture.setCause(future.cause()); - responseFuture.putResponse(null); - try { - responseFuture.executeInvokeCallback(); - } catch (Exception ex) { - log.error("execute callback error", ex); - } finally { - responseFuture.release(); - } - }); - } catch (Exception ex) { - responseFuture.release(); - throw new RemotingException(String.format("Send transporter to host: %s failed", host), ex); - } - } else { - String message = String.format( - "try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", - timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); - throw new RemotingTooMuchRequestException(message); - } - } - - public IRpcResponse sendSync(final Host host, final Transporter transporter, + public IRpcResponse sendSync(final Host host, + final Transporter transporter, final long timeoutMillis) throws InterruptedException, RemotingException { - final Channel channel = getChannel(host); + final Channel channel = getOrCreateChannel(host); if (channel == null) { throw new RemotingException(String.format("connect to : %s fail", host)); } final long opaque = transporter.getHeader().getOpaque(); - final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); + final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis); channel.writeAndFlush(transporter).addListener(future -> { if (future.isSuccess()) { responseFuture.setSendOk(true); @@ -220,39 +141,44 @@ public class NettyRemotingClient implements AutoCloseable { return iRpcResponse; } - public Channel getChannel(Host host) { + Channel getOrCreateChannel(Host host) { Channel channel = channels.get(host); if (channel != null && channel.isActive()) { return channel; } - return createChannel(host, true); + try { + channelsLock.lock(); + channel = channels.get(host); + if (channel != null && channel.isActive()) { + return channel; + } + channel = createChannel(host); + channels.put(host, channel); + } finally { + channelsLock.unlock(); + } + return channel; } /** * create channel * - * @param host host - * @param isSync sync flag + * @param host host * @return channel */ - public Channel createChannel(Host host, boolean isSync) { - ChannelFuture future; + Channel createChannel(Host host) { try { - synchronized (bootstrap) { - future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); - } - if (isSync) { - future.sync(); - } + ChannelFuture future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); + future = future.sync(); if (future.isSuccess()) { - Channel channel = future.channel(); - channels.put(host, channel); - return channel; + return future.channel(); + } else { + throw new IllegalArgumentException("connect to host: " + host + " failed", future.cause()); } - } catch (Exception ex) { - log.warn(String.format("connect to %s error", host), ex); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Connect to host: " + host + " failed", e); } - return null; } @Override @@ -263,12 +189,6 @@ public class NettyRemotingClient implements AutoCloseable { if (workerGroup != null) { this.workerGroup.shutdownGracefully(); } - if (callbackExecutor != null) { - this.callbackExecutor.shutdownNow(); - } - if (this.responseFutureExecutor != null) { - this.responseFutureExecutor.shutdownNow(); - } log.info("netty client closed"); } catch (Exception ex) { log.error("netty client close exception", ex); @@ -277,16 +197,23 @@ public class NettyRemotingClient implements AutoCloseable { } private void closeChannels() { - for (Channel channel : this.channels.values()) { - channel.close(); + try { + channelsLock.lock(); + channels.values().forEach(Channel::close); + } finally { + channelsLock.unlock(); } - this.channels.clear(); } public void closeChannel(Host host) { - Channel channel = this.channels.remove(host); - if (channel != null) { - channel.close(); + try { + channelsLock.lock(); + Channel channel = this.channels.remove(host); + if (channel != null) { + channel.close(); + } + } finally { + channelsLock.unlock(); } } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java similarity index 95% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java index 7bbebfbf3d..d14a8aa54e 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java index 28d82532be..44d310e70b 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClientFactory; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; public class SingletonJdkDynamicRpcClientProxyFactory { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java index b5fdf3fb71..4731a22d0a 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.extract.base.IRpcResponse; -import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException; @@ -29,9 +28,9 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import java.lang.reflect.Method; -public class SyncClientMethodInvoker extends BaseRemoteMethodInvoker { +class SyncClientMethodInvoker extends AbstractClientMethodInvoker { - public SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { super(serverHost, localMethod, nettyRemotingClient); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java index 35405c5578..1fbbd9ed6c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.extract.base.future; import org.apache.dolphinscheduler.extract.base.IRpcResponse; -import java.util.Iterator; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -34,17 +32,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class ResponseFuture { - private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); + private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(); private final long opaque; // remove the timeout private final long timeoutMillis; - private final InvokeCallback invokeCallback; - - private final ReleaseSemaphore releaseSemaphore; - private final CountDownLatch latch = new CountDownLatch(1); private final long beginTimestamp = System.currentTimeMillis(); @@ -57,14 +51,9 @@ public class ResponseFuture { private Throwable cause; - public ResponseFuture(long opaque, - long timeoutMillis, - InvokeCallback invokeCallback, - ReleaseSemaphore releaseSemaphore) { + public ResponseFuture(long opaque, long timeoutMillis) { this.opaque = opaque; this.timeoutMillis = timeoutMillis; - this.invokeCallback = invokeCallback; - this.releaseSemaphore = releaseSemaphore; FUTURE_TABLE.put(opaque, this); } @@ -90,10 +79,6 @@ public class ResponseFuture { return FUTURE_TABLE.get(opaque); } - public void removeFuture() { - FUTURE_TABLE.remove(opaque); - } - /** * whether timeout * @@ -104,15 +89,6 @@ public class ResponseFuture { return diff > this.timeoutMillis; } - /** - * execute invoke callback - */ - public void executeInvokeCallback() { - if (invokeCallback != null) { - invokeCallback.operationComplete(this); - } - } - public boolean isSendOK() { return sendOk; } @@ -129,52 +105,4 @@ public class ResponseFuture { return cause; } - public long getOpaque() { - return opaque; - } - - public long getTimeoutMillis() { - return timeoutMillis; - } - - public long getBeginTimestamp() { - return beginTimestamp; - } - - public InvokeCallback getInvokeCallback() { - return invokeCallback; - } - - /** - * release - */ - public void release() { - if (this.releaseSemaphore != null) { - this.releaseSemaphore.release(); - } - } - - /** - * scan future table - */ - public static void scanFutureTable() { - Iterator> it = FUTURE_TABLE.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry next = it.next(); - ResponseFuture future = next.getValue(); - if ((future.getBeginTimestamp() + future.getTimeoutMillis() + 1000) > System.currentTimeMillis()) { - continue; - } - try { - // todo: use thread pool to execute the async callback, otherwise will block the scan thread - future.release(); - future.executeInvokeCallback(); - } catch (Exception ex) { - log.error("ScanFutureTable, execute callback error, requestId: {}", future.getOpaque(), ex); - } - it.remove(); - log.debug("Remove timeout request: {}", future); - } - } - } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java index b4978172f1..f57ff0b609 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.extract.base.server; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; @@ -30,6 +29,7 @@ import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import lombok.extern.slf4j.Slf4j; @@ -42,14 +42,14 @@ import io.netty.handler.timeout.IdleStateEvent; @Slf4j @ChannelHandler.Sharable -public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { +class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { - private final NettyRemotingServer nettyRemotingServer; + private final ExecutorService methodInvokeExecutor; private final Map methodInvokerMap; - public JdkDynamicServerHandler(NettyRemotingServer nettyRemotingServer) { - this.nettyRemotingServer = nettyRemotingServer; + JdkDynamicServerHandler(ExecutorService methodInvokeExecutor) { + this.methodInvokeExecutor = methodInvokeExecutor; this.methodInvokerMap = new ConcurrentHashMap<>(); } @@ -90,7 +90,7 @@ public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { channel.writeAndFlush(response); return; } - nettyRemotingServer.getDefaultExecutor().execute(() -> { + methodInvokeExecutor.execute(() -> { StandardRpcResponse iRpcResponse; try { StandardRpcRequest standardRpcRequest = diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java similarity index 75% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index 365a17dd03..9beeaced3d 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -15,15 +15,13 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.exception.RemoteException; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; -import org.apache.dolphinscheduler.extract.base.server.JdkDynamicServerHandler; -import org.apache.dolphinscheduler.extract.base.server.ServerMethodInvoker; import org.apache.dolphinscheduler.extract.base.utils.Constants; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; @@ -32,6 +30,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; @@ -48,12 +47,15 @@ import io.netty.handler.timeout.IdleStateHandler; * remoting netty server */ @Slf4j -public class NettyRemotingServer { +class NettyRemotingServer { private final ServerBootstrap serverBootstrap = new ServerBootstrap(); - private final ExecutorService defaultExecutor = ThreadUtils - .newDaemonFixedThreadExecutor("NettyRemotingServerThread", Runtime.getRuntime().availableProcessors() * 2); + @Getter + private final String serverName; + + @Getter + private final ExecutorService methodInvokerExecutor; private final EventLoopGroup bossGroup; @@ -61,16 +63,20 @@ public class NettyRemotingServer { private final NettyServerConfig serverConfig; - private final JdkDynamicServerHandler serverHandler = new JdkDynamicServerHandler(this); + private final JdkDynamicServerHandler channelHandler; private final AtomicBoolean isStarted = new AtomicBoolean(false); - public NettyRemotingServer(final NettyServerConfig serverConfig) { + NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; + this.serverName = serverConfig.getServerName(); + this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor( + serverName + "MethodInvoker-%d", Runtime.getRuntime().availableProcessors() * 2 + 1); + this.channelHandler = new JdkDynamicServerHandler(methodInvokerExecutor); ThreadFactory bossThreadFactory = - ThreadUtils.newDaemonThreadFactory(serverConfig.getServerName() + "BossThread_%s"); + ThreadUtils.newDaemonThreadFactory(serverName + "BossThread-%d"); ThreadFactory workerThreadFactory = - ThreadUtils.newDaemonThreadFactory(serverConfig.getServerName() + "WorkerThread_%s"); + ThreadUtils.newDaemonThreadFactory(serverName + "WorkerThread-%d"); if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory); @@ -80,7 +86,7 @@ public class NettyRemotingServer { } } - public void start() { + void start() { if (isStarted.compareAndSet(false, true)) { this.serverBootstrap .group(this.bossGroup, this.workGroup) @@ -103,9 +109,9 @@ public class NettyRemotingServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - log.error("{} bind fail {}, exit", serverConfig.getServerName(), e.getMessage(), e); throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort())); + String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), + e); } if (future.isSuccess()) { @@ -113,14 +119,9 @@ public class NettyRemotingServer { return; } - if (future.cause() != null) { - throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), - future.cause()); - } else { - throw new RemoteException( - String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort())); - } + throw new RemoteException( + String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()), + future.cause()); } } @@ -135,18 +136,14 @@ public class NettyRemotingServer { .addLast("decoder", new TransporterDecoder()) .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) - .addLast("handler", serverHandler); + .addLast("handler", channelHandler); } - public ExecutorService getDefaultExecutor() { - return defaultExecutor; + void registerMethodInvoker(ServerMethodInvoker methodInvoker) { + channelHandler.registerMethodInvoker(methodInvoker); } - public void registerMethodInvoker(ServerMethodInvoker methodInvoker) { - serverHandler.registerMethodInvoker(methodInvoker); - } - - public void close() { + void close() { if (isStarted.compareAndSet(true, false)) { try { if (bossGroup != null) { @@ -155,7 +152,7 @@ public class NettyRemotingServer { if (workGroup != null) { this.workGroup.shutdownGracefully(); } - defaultExecutor.shutdown(); + methodInvokerExecutor.shutdown(); } catch (Exception ex) { log.error("netty server close exception", ex); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java similarity index 84% rename from dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java index 6bf1b8d31c..70ed0529e8 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.extract.base; +package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import lombok.experimental.UtilityClass; @UtilityClass -public class NettyRemotingServerFactory { +class NettyRemotingServerFactory { - public NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { + NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { return new NettyRemotingServer(nettyServerConfig); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java new file mode 100644 index 0000000000..213868ba46 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; + +import java.lang.reflect.Method; + +import lombok.extern.slf4j.Slf4j; + +/** + * The RpcServer based on Netty. The server will register the method invoker and provide the service to the client. + * Once the server is started, it will listen on the port and wait for the client to connect. + * 

+ *          RpcServer rpcServer = new RpcServer(new NettyServerConfig());
+ *          rpcServer.registerServerMethodInvokerProvider(new ServerMethodInvokerProviderImpl());
+ *          rpcServer.start();
+ *
+ */ +@Slf4j +public class RpcServer implements ServerMethodInvokerRegistry, AutoCloseable { + + private final NettyRemotingServer nettyRemotingServer; + + public RpcServer(NettyServerConfig nettyServerConfig) { + this.nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig); + } + + public void start() { + nettyRemotingServer.start(); + } + + @Override + public void registerServerMethodInvokerProvider(Object serverMethodInvokerProviderBean) { + for (Class anInterface : serverMethodInvokerProviderBean.getClass().getInterfaces()) { + if (anInterface.getAnnotation(RpcService.class) == null) { + continue; + } + for (Method method : anInterface.getDeclaredMethods()) { + RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class); + if (rpcMethod == null) { + continue; + } + ServerMethodInvoker serverMethodInvoker = + new ServerMethodInvokerImpl(serverMethodInvokerProviderBean, method); + nettyRemotingServer.registerMethodInvoker(serverMethodInvoker); + log.debug("Register ServerMethodInvoker: {} to bean: {}", + serverMethodInvoker.getMethodIdentify(), serverMethodInvoker.getMethodProviderIdentify()); + } + } + } + + @Override + public void close() { + nettyRemotingServer.close(); + } +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java index ee633217b2..151b54bb97 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java @@ -17,10 +17,12 @@ package org.apache.dolphinscheduler.extract.base.server; -public interface ServerMethodInvoker { +interface ServerMethodInvoker { String getMethodIdentify(); + String getMethodProviderIdentify(); + Object invoke(final Object... arg) throws Throwable; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java index eea9da5e14..4c29650aa0 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.extract.base.server; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -public class ServerMethodInvokerImpl implements ServerMethodInvoker { +class ServerMethodInvokerImpl implements ServerMethodInvoker { private final Object serviceBean; @@ -28,7 +28,7 @@ public class ServerMethodInvokerImpl implements ServerMethodInvoker { private final String methodIdentify; - public ServerMethodInvokerImpl(Object serviceBean, Method method) { + ServerMethodInvokerImpl(Object serviceBean, Method method) { this.serviceBean = serviceBean; this.method = method; this.methodIdentify = method.toGenericString(); @@ -48,4 +48,9 @@ public class ServerMethodInvokerImpl implements ServerMethodInvoker { public String getMethodIdentify() { return methodIdentify; } + + @Override + public String getMethodProviderIdentify() { + return serviceBean.getClass().getName(); + } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java new file mode 100644 index 0000000000..4e56be2617 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerRegistry.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +interface ServerMethodInvokerRegistry { + + /** + * Register service object, which will be used to invoke the {@link ServerMethodInvoker}. + * The serverMethodInvokerProviderObject should implement with interface which contains {@link org.apache.dolphinscheduler.extract.base.RpcService} annotation. + */ + void registerServerMethodInvokerProvider(Object serverMethodInvokerProviderObject); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java index 2b87a70080..de4943990c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java @@ -17,11 +17,7 @@ package org.apache.dolphinscheduler.extract.base.server; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; -import org.apache.dolphinscheduler.extract.base.RpcMethod; -import org.apache.dolphinscheduler.extract.base.RpcService; - -import java.lang.reflect.Method; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import lombok.extern.slf4j.Slf4j; @@ -29,38 +25,21 @@ import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; import org.springframework.lang.Nullable; +/** + * The RpcServer which will auto discovery the {@link ServerMethodInvoker} from Spring container. + */ @Slf4j -public class SpringServerMethodInvokerDiscovery implements BeanPostProcessor { +public class SpringServerMethodInvokerDiscovery extends RpcServer implements BeanPostProcessor { - protected final NettyRemotingServer nettyRemotingServer; - - public SpringServerMethodInvokerDiscovery(NettyRemotingServer nettyRemotingServer) { - this.nettyRemotingServer = nettyRemotingServer; + public SpringServerMethodInvokerDiscovery(NettyServerConfig nettyServerConfig) { + super(nettyServerConfig); } @Nullable @Override public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - Class[] interfaces = bean.getClass().getInterfaces(); - for (Class anInterface : interfaces) { - if (anInterface.getAnnotation(RpcService.class) == null) { - continue; - } - registerRpcMethodInvoker(anInterface, bean, beanName); - } + registerServerMethodInvokerProvider(bean); return bean; } - private void registerRpcMethodInvoker(Class anInterface, Object bean, String beanName) { - Method[] declaredMethods = anInterface.getDeclaredMethods(); - for (Method method : declaredMethods) { - RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class); - if (rpcMethod == null) { - continue; - } - ServerMethodInvoker methodInvoker = new ServerMethodInvokerImpl(bean, method); - nettyRemotingServer.registerMethodInvoker(methodInvoker); - log.debug("Register ServerMethodInvoker: {} to bean: {}", methodInvoker.getMethodIdentify(), beanName); - } - } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java index 521cf7c75a..92ed49934c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.extract.base.client; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.RpcService; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; @@ -37,7 +36,7 @@ import org.junit.jupiter.api.Test; public class SingletonJdkDynamicRpcClientProxyFactoryTest { - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private String serverAddress; @@ -48,11 +47,10 @@ public class SingletonJdkDynamicRpcClientProxyFactoryTest { .serverName("ApiServer") .listenPort(listenPort) .build(); - nettyRemotingServer = new NettyRemotingServer(nettyServerConfig); - nettyRemotingServer.start(); serverAddress = "localhost:" + listenPort; - new SpringServerMethodInvokerDiscovery(nettyRemotingServer) - .postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); + springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new IServiceImpl()); + springServerMethodInvokerDiscovery.start(); } @Test @@ -82,7 +80,7 @@ public class SingletonJdkDynamicRpcClientProxyFactoryTest { @AfterEach public void tearDown() { - nettyRemotingServer.close(); + springServerMethodInvokerDiscovery.close(); } @RpcService diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java new file mode 100644 index 0000000000..5bd1a0a0ba --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/utils/HostTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.utils; + +import org.junit.jupiter.api.Test; + +import com.google.common.truth.Truth; + +class HostTest { + + @Test + void testEquals() { + Truth.assertThat(Host.of("localhost:8080")).isEqualTo(Host.of("localhost:8080")); + } + +} diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml index d521e53bc2..f069b07079 100644 --- a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml +++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml @@ -56,6 +56,13 @@ conf + + ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources + + **/*.yaml + + conf + diff --git a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh index c183f31bb1..1171cd7804 100644 --- a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh +++ b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh @@ -24,6 +24,7 @@ -XX:+PrintGCDetails -Xloggc:gc.log +-XX:-OmitStackTraceInFastThrow -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 7988570135..9b507500b9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; @@ -108,6 +109,7 @@ public class MasterServer implements IStoppable { // install task plugin TaskPluginManager.loadPlugin(); + DataSourceProcessorProvider.initialize(); this.masterSlotManager.start(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java index 832c1b336b..5990a53e0f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java @@ -66,7 +66,6 @@ public class TaskExecutionContextBuilder { taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); taskExecutionContext.setEnvironmentConfig(taskInstance.getEnvironmentConfig()); taskExecutionContext.setHost(taskInstance.getHost()); - taskExecutionContext.setDelayTime(taskInstance.getDelayTime()); taskExecutionContext.setVarPool(taskInstance.getVarPool()); taskExecutionContext.setDryRun(taskInstance.getDryRun()); taskExecutionContext.setTestFlag(taskInstance.getTestFlag()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java index 08ee6b66c5..3e6f3d4ada 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java @@ -28,10 +28,9 @@ public interface StateEventHandler { * @param stateEvent given state event. * @throws StateEventHandleException this exception means it can be recovered. * @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop. - * @throws StateEventHandleException this means it can be recovered. */ boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, - StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError, StateEventHandleFailure; + StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError; StateEventType getEventType(); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java index 0eaf885d11..ab89b021d6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.rpc; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -31,21 +30,8 @@ import org.springframework.stereotype.Component; public class MasterRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { public MasterRpcServer(MasterConfig masterConfig) { - super(NettyRemotingServerFactory.buildNettyRemotingServer(NettyServerConfig.builder() - .serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()).build())); - } - - public void start() { - log.info("Starting MasterRPCServer..."); - nettyRemotingServer.start(); - log.info("Started MasterRPCServer..."); - } - - @Override - public void close() { - log.info("Closing MasterRPCServer..."); - nettyRemotingServer.close(); - log.info("Closed MasterRPCServer..."); + super(NettyServerConfig.builder().serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()) + .build()); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskExecuteRunnable.java index fefdbf3493..2e41173c4b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskExecuteRunnable.java @@ -52,4 +52,32 @@ public abstract class BaseTaskExecuteRunnable implements TaskExecuteRunnable { return taskExecutionContext; } + @Override + public int compareTo(TaskExecuteRunnable other) { + if (other == null) { + return 1; + } + int workflowInstancePriorityCompareResult = workflowInstance.getProcessInstancePriority().getCode() - + other.getWorkflowInstance().getProcessInstancePriority().getCode(); + if (workflowInstancePriorityCompareResult != 0) { + return workflowInstancePriorityCompareResult; + } + + // smaller number, higher priority + int taskInstancePriorityCompareResult = taskInstance.getTaskInstancePriority().getCode() + - other.getTaskInstance().getTaskInstancePriority().getCode(); + if (taskInstancePriorityCompareResult != 0) { + return taskInstancePriorityCompareResult; + } + + // larger number, higher priority + int taskGroupPriorityCompareResult = + taskInstance.getTaskGroupPriority() - other.getTaskInstance().getTaskGroupPriority(); + if (taskGroupPriorityCompareResult != 0) { + return -taskGroupPriorityCompareResult; + } + // earlier submit time, higher priority + return taskInstance.getFirstSubmitTime().compareTo(other.getTaskInstance().getFirstSubmitTime()); + } + } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/DefaultTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/DefaultTaskExecuteRunnable.java index c1b13717bd..ba4f447216 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/DefaultTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/DefaultTaskExecuteRunnable.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; -public class DefaultTaskExecuteRunnable extends PriorityDelayTaskExecuteRunnable { +public class DefaultTaskExecuteRunnable extends BaseTaskExecuteRunnable { private final TaskExecuteRunnableOperatorManager taskExecuteRunnableOperatorManager; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueue.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueue.java index f03bd6b903..21d6c890f5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueue.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueue.java @@ -17,7 +17,8 @@ package org.apache.dolphinscheduler.server.master.runner; -import java.util.concurrent.DelayQueue; +import org.apache.dolphinscheduler.server.master.runner.queue.DelayEntry; +import org.apache.dolphinscheduler.server.master.runner.queue.PriorityDelayQueue; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -25,26 +26,42 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; /** - * The class is used to store {@link TaskExecuteRunnable} which needs to be dispatched. The {@link TaskExecuteRunnable} will be stored in a {@link DelayQueue}, - * if the {@link TaskExecuteRunnable}'s delay time is 0, then it will be consumed by {@link GlobalTaskDispatchWaitingQueueLooper}. + * The class is used to store {@link TaskExecuteRunnable} which needs to be dispatched. The {@link TaskExecuteRunnable} + * will be stored in {@link PriorityDelayQueue}, if the {@link TaskExecuteRunnable}'s delay time is 0, then it will be + * consumed by {@link GlobalTaskDispatchWaitingQueueLooper}. + *

+ * The order of {@link TaskExecuteRunnable} in the {@link PriorityDelayQueue} is determined by {@link TaskExecuteRunnable#compareTo}. */ @Slf4j @Component public class GlobalTaskDispatchWaitingQueue { - private final DelayQueue queue = new DelayQueue<>(); + private final PriorityDelayQueue> priorityDelayQueue = new PriorityDelayQueue<>(); - public void submitTaskExecuteRunnable(DefaultTaskExecuteRunnable priorityTaskExecuteRunnable) { - queue.put(priorityTaskExecuteRunnable); + /** + * Submit a {@link TaskExecuteRunnable} with delay time 0, it will be consumed immediately. + */ + public void dispatchTaskExecuteRunnable(TaskExecuteRunnable taskExecuteRunnable) { + dispatchTaskExecuteRunnableWithDelay(taskExecuteRunnable, 0); } + /** + * Submit a {@link TaskExecuteRunnable} with delay time, if the delay time <= 0 then it can be consumed. + */ + public void dispatchTaskExecuteRunnableWithDelay(TaskExecuteRunnable taskExecuteRunnable, long delayTimeMills) { + priorityDelayQueue.add(new DelayEntry<>(delayTimeMills, taskExecuteRunnable)); + } + + /** + * Consume {@link TaskExecuteRunnable} from the {@link PriorityDelayQueue}, only the delay time <= 0 can be consumed. + */ @SneakyThrows - public DefaultTaskExecuteRunnable takeTaskExecuteRunnable() { - return queue.take(); + public TaskExecuteRunnable takeTaskExecuteRunnable() { + return priorityDelayQueue.take().getData(); } public int getWaitingDispatchTaskNumber() { - return queue.size(); + return priorityDelayQueue.size(); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java index eabbdd8e10..5cfb285c28 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.server.master.runner; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.master.runner.dispatcher.TaskDispatchFactory; -import org.apache.dolphinscheduler.server.master.runner.dispatcher.TaskDispatcher; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; @@ -43,10 +41,6 @@ public class GlobalTaskDispatchWaitingQueueLooper extends BaseDaemonThread imple private final AtomicBoolean RUNNING_FLAG = new AtomicBoolean(false); - private final AtomicInteger DISPATCHED_CONSECUTIVE_FAILURE_TIMES = new AtomicInteger(); - - private static final Integer MAX_DISPATCHED_FAILED_TIMES = 100; - public GlobalTaskDispatchWaitingQueueLooper() { super("GlobalTaskDispatchWaitingQueueLooper"); } @@ -64,29 +58,34 @@ public class GlobalTaskDispatchWaitingQueueLooper extends BaseDaemonThread imple @Override public void run() { - DefaultTaskExecuteRunnable defaultTaskExecuteRunnable; while (RUNNING_FLAG.get()) { - defaultTaskExecuteRunnable = globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable(); - try { - TaskExecutionStatus status = defaultTaskExecuteRunnable.getTaskInstance().getState(); - if (status != TaskExecutionStatus.SUBMITTED_SUCCESS && status != TaskExecutionStatus.DELAY_EXECUTION) { - log.warn("The TaskInstance {} state is : {}, will not dispatch", - defaultTaskExecuteRunnable.getTaskInstance().getName(), status); - continue; - } + doDispatch(); + } + } - TaskDispatcher taskDispatcher = - taskDispatchFactory.getTaskDispatcher(defaultTaskExecuteRunnable.getTaskInstance()); - taskDispatcher.dispatchTask(defaultTaskExecuteRunnable); - DISPATCHED_CONSECUTIVE_FAILURE_TIMES.set(0); - } catch (Exception e) { - defaultTaskExecuteRunnable.getTaskExecutionContext().increaseDispatchFailTimes(); - globalTaskDispatchWaitingQueue.submitTaskExecuteRunnable(defaultTaskExecuteRunnable); - if (DISPATCHED_CONSECUTIVE_FAILURE_TIMES.incrementAndGet() > MAX_DISPATCHED_FAILED_TIMES) { - ThreadUtils.sleep(10 * 1000L); - } - log.error("Dispatch Task: {} failed", defaultTaskExecuteRunnable.getTaskInstance().getName(), e); + void doDispatch() { + final TaskExecuteRunnable taskExecuteRunnable = globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable(); + TaskInstance taskInstance = taskExecuteRunnable.getTaskInstance(); + if (taskInstance == null) { + // This case shouldn't happen, but if it does, log an error and continue + log.error("The TaskInstance is null, drop it(This case shouldn't happen)"); + return; + } + try { + TaskExecutionStatus status = taskInstance.getState(); + if (status != TaskExecutionStatus.SUBMITTED_SUCCESS && status != TaskExecutionStatus.DELAY_EXECUTION) { + log.warn("The TaskInstance {} state is : {}, will not dispatch", taskInstance.getName(), status); + return; } + taskDispatchFactory.getTaskDispatcher(taskInstance).dispatchTask(taskExecuteRunnable); + } catch (Exception e) { + // If dispatch failed, will put the task back to the queue + // The task will be dispatched after waiting time. + // the waiting time will increase multiple of times, but will not exceed 60 seconds + long waitingTimeMills = Math.max( + taskExecuteRunnable.getTaskExecutionContext().increaseDispatchFailTimes() * 1_000L, 60_000L); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnableWithDelay(taskExecuteRunnable, waitingTimeMills); + log.error("Dispatch Task: {} failed will retry after: {}/ms", taskInstance.getName(), waitingTimeMills, e); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/PriorityDelayTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/PriorityDelayTaskExecuteRunnable.java deleted file mode 100644 index 255ec6c8ac..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/PriorityDelayTaskExecuteRunnable.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.runner; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; - -import java.util.concurrent.Delayed; -import java.util.concurrent.TimeUnit; - -public abstract class PriorityDelayTaskExecuteRunnable extends BaseTaskExecuteRunnable implements Delayed { - - public PriorityDelayTaskExecuteRunnable(ProcessInstance workflowInstance, - TaskInstance taskInstance, - TaskExecutionContext taskExecutionContext) { - super(workflowInstance, taskInstance, taskExecutionContext); - } - - @Override - public long getDelay(TimeUnit unit) { - return unit.convert( - DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), - taskExecutionContext.getDelayTime() * 60L), - TimeUnit.SECONDS); - } - - @Override - public int compareTo(Delayed o) { - if (o == null) { - return 1; - } - int delayTimeCompareResult = - Long.compare(this.getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); - if (delayTimeCompareResult != 0) { - return delayTimeCompareResult; - } - PriorityDelayTaskExecuteRunnable other = (PriorityDelayTaskExecuteRunnable) o; - // the smaller dispatch fail times, the higher priority - int dispatchFailTimesCompareResult = taskExecutionContext.getDispatchFailTimes() - - other.getTaskExecutionContext().getDispatchFailTimes(); - if (dispatchFailTimesCompareResult != 0) { - return dispatchFailTimesCompareResult; - } - int workflowInstancePriorityCompareResult = workflowInstance.getProcessInstancePriority().getCode() - - other.getWorkflowInstance().getProcessInstancePriority().getCode(); - if (workflowInstancePriorityCompareResult != 0) { - return workflowInstancePriorityCompareResult; - } - long workflowInstanceIdCompareResult = workflowInstance.getId().compareTo(other.getWorkflowInstance().getId()); - if (workflowInstanceIdCompareResult != 0) { - return workflowInstancePriorityCompareResult; - } - int taskInstancePriorityCompareResult = taskInstance.getTaskInstancePriority().getCode() - - other.getTaskInstance().getTaskInstancePriority().getCode(); - if (taskInstancePriorityCompareResult != 0) { - return taskInstancePriorityCompareResult; - } - // larger number, higher priority - int taskGroupPriorityCompareResult = - taskInstance.getTaskGroupPriority() - other.getTaskInstance().getTaskGroupPriority(); - if (taskGroupPriorityCompareResult != 0) { - return -taskGroupPriorityCompareResult; - } - // The task instance shouldn't be equals - return taskInstance.getId().compareTo(other.getTaskInstance().getId()); - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnable.java index 8f66189617..62617f4aac 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnable.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; * This interface is used to define a task which is executing. * todo: split to MasterTaskExecuteRunnable and WorkerTaskExecuteRunnable */ -public interface TaskExecuteRunnable { +public interface TaskExecuteRunnable extends Comparable { void dispatch(); @@ -40,4 +40,5 @@ public interface TaskExecuteRunnable { TaskInstance getTaskInstance(); TaskExecutionContext getTaskExecutionContext(); + } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index eafba17f69..c35fd9991f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -57,16 +57,14 @@ import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.StateEventHandleError; import org.apache.dolphinscheduler.server.master.event.StateEventHandleException; -import org.apache.dolphinscheduler.server.master.event.StateEventHandleFailure; import org.apache.dolphinscheduler.server.master.event.StateEventHandler; import org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; @@ -97,6 +95,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -298,13 +297,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { stateEvent, stateEventHandleException); ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); - } catch (StateEventHandleFailure stateEventHandleFailure) { - log.error("State event handle failed, will move event to the tail: {}", - stateEvent, - stateEventHandleFailure); - this.stateEvents.remove(stateEvent); - this.stateEvents.offer(stateEvent); - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); } catch (Exception e) { // we catch the exception here, since if the state event handle failed, the state event will still // keep @@ -376,7 +368,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (taskInstance.getState().isSuccess()) { completeTaskSet.add(taskInstance.getTaskCode()); - mergeTaskInstanceVarPool(taskInstance); + workflowInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString( + Lists.newArrayList(workflowInstance.getVarPool(), taskInstance.getVarPool()))); processInstanceDao.upsertProcessInstance(workflowInstance); ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(workflowInstance.getId()); @@ -441,7 +434,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { /** * crate new task instance to retry, different objects from the original - * */ private void retryTaskInstance(TaskInstance taskInstance) throws StateEventHandleException { ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); @@ -532,16 +524,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } } - /** - * check if task instance exist by id - */ - public boolean checkTaskInstanceById(int taskInstanceId) { - if (taskInstanceMap.isEmpty()) { - return false; - } - return taskInstanceMap.containsKey(taskInstanceId); - } - /** * get task instance from memory */ @@ -657,6 +639,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { command.setProcessInstanceId(0); command.setProcessDefinitionVersion(workflowInstance.getProcessDefinitionVersion()); command.setTestFlag(workflowInstance.getTestFlag()); + command.setTenantCode(workflowInstance.getTenantCode()); int create = commandService.createCommand(command); processService.saveCommandTrigger(command.getId(), workflowInstance.getId()); return create; @@ -1070,7 +1053,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { * new a taskInstance * * @param processInstance process instance - * @param taskNode task node + * @param taskNode task node * @return task instance */ public TaskInstance newTaskInstance(ProcessInstance processInstance, TaskNode taskNode) { @@ -1161,80 +1144,32 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { return taskInstance; } - public void getPreVarPool(TaskInstance taskInstance, Set preTask) { + void initializeTaskInstanceVarPool(TaskInstance taskInstance) { + // get pre task ,get all the task varPool to this task + // Do not use dag.getPreviousNodes because of the dag may be miss the upstream node + String preTasks = + workflowExecuteContext.getWorkflowGraph().getTaskNodeByCode(taskInstance.getTaskCode()).getPreTasks(); + Set preTaskList = new HashSet<>(JSONUtils.toList(preTasks, Long.class)); ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); - Map allProperty = new HashMap<>(); - Map allTaskInstance = new HashMap<>(); - if (CollectionUtils.isNotEmpty(preTask)) { - for (Long preTaskCode : preTask) { - Optional existTaskInstanceOptional = getTaskInstance(preTaskCode); - if (!existTaskInstanceOptional.isPresent()) { - continue; - } - Integer taskId = existTaskInstanceOptional.get().getId(); - if (taskId == null) { - continue; - } - TaskInstance preTaskInstance = taskInstanceMap.get(taskId); - if (preTaskInstance == null) { - continue; - } - String preVarPool = preTaskInstance.getVarPool(); - if (StringUtils.isNotEmpty(preVarPool)) { - List properties = JSONUtils.toList(preVarPool, Property.class); - for (Property info : properties) { - setVarPoolValue(allProperty, allTaskInstance, preTaskInstance, info); - } - } - } - if (allProperty.size() > 0) { - taskInstance.setVarPool(JSONUtils.toJsonString(allProperty.values())); - } - } else { - if (StringUtils.isNotEmpty(workflowInstance.getVarPool())) { - taskInstance.setVarPool(workflowInstance.getVarPool()); - } + if (CollectionUtils.isEmpty(preTaskList)) { + taskInstance.setVarPool(workflowInstance.getVarPool()); + return; } + List preTaskInstanceVarPools = preTaskList + .stream() + .map(taskCode -> getTaskInstance(taskCode).orElse(null)) + .filter(Objects::nonNull) + .sorted(Comparator.comparing(TaskInstance::getEndTime)) + .map(TaskInstance::getVarPool) + .collect(Collectors.toList()); + taskInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString(preTaskInstanceVarPools)); } public Collection getAllTaskInstances() { return taskInstanceMap.values(); } - private void setVarPoolValue(Map allProperty, - Map allTaskInstance, - TaskInstance preTaskInstance, Property thisProperty) { - // for this taskInstance all the param in this part is IN. - thisProperty.setDirect(Direct.IN); - // get the pre taskInstance Property's name - String proName = thisProperty.getProp(); - // if the Previous nodes have the Property of same name - if (allProperty.containsKey(proName)) { - // comparison the value of two Property - Property otherPro = allProperty.get(proName); - // if this property'value of loop is empty,use the other,whether the other's value is empty or not - if (StringUtils.isEmpty(thisProperty.getValue())) { - allProperty.put(proName, otherPro); - // if property'value of loop is not empty,and the other's value is not empty too, use the earlier value - } else if (StringUtils.isNotEmpty(otherPro.getValue())) { - TaskInstance otherTask = allTaskInstance.get(proName); - if (otherTask.getEndTime().getTime() > preTaskInstance.getEndTime().getTime()) { - allProperty.put(proName, thisProperty); - allTaskInstance.put(proName, preTaskInstance); - } else { - allProperty.put(proName, otherPro); - } - } else { - allProperty.put(proName, thisProperty); - allTaskInstance.put(proName, preTaskInstance); - } - } else { - allProperty.put(proName, thisProperty); - allTaskInstance.put(proName, preTaskInstance); - } - } - /** * get complete task instance map, taskCode as key */ @@ -1311,45 +1246,10 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } // the end node of the branch of the dag if (parentNodeCode != null && dag.getEndNode().contains(parentNodeCode)) { - Optional existTaskInstanceOptional = getTaskInstance(parentNodeCode); - if (existTaskInstanceOptional.isPresent()) { - TaskInstance endTaskInstance = taskInstanceMap.get(existTaskInstanceOptional.get().getId()); - String taskInstanceVarPool = endTaskInstance.getVarPool(); - if (StringUtils.isNotEmpty(taskInstanceVarPool)) { - Set taskProperties = new HashSet<>(JSONUtils.toList(taskInstanceVarPool, Property.class)); - String processInstanceVarPool = workflowInstance.getVarPool(); - List processGlobalParams = - new ArrayList<>(JSONUtils.toList(workflowInstance.getGlobalParams(), Property.class)); - Map oldProcessGlobalParamsMap = processGlobalParams.stream() - .collect(Collectors.toMap(Property::getProp, Property::getDirect)); - Set processVarPoolOut = taskProperties.stream() - .filter(property -> property.getDirect().equals(Direct.OUT) - && oldProcessGlobalParamsMap.containsKey(property.getProp()) - && oldProcessGlobalParamsMap.get(property.getProp()).equals(Direct.OUT)) - .collect(Collectors.toSet()); - Set taskVarPoolIn = - taskProperties.stream().filter(property -> property.getDirect().equals(Direct.IN)) - .collect(Collectors.toSet()); - if (StringUtils.isNotEmpty(processInstanceVarPool)) { - Set properties = - new HashSet<>(JSONUtils.toList(processInstanceVarPool, Property.class)); - Set newProcessVarPoolKeys = - taskProperties.stream().map(Property::getProp).collect(Collectors.toSet()); - properties = properties.stream() - .filter(property -> !newProcessVarPoolKeys.contains(property.getProp())) - .collect(Collectors.toSet()); - properties.addAll(processVarPoolOut); - properties.addAll(taskVarPoolIn); + getTaskInstance(parentNodeCode) + .ifPresent(endTaskInstance -> workflowInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString( + Lists.newArrayList(workflowInstance.getVarPool(), endTaskInstance.getVarPool())))); - workflowInstance.setVarPool(JSONUtils.toJsonString(properties)); - } else { - Set varPool = new HashSet<>(); - varPool.addAll(taskVarPoolIn); - varPool.addAll(processVarPoolOut); - workflowInstance.setVarPool(JSONUtils.toJsonString(varPool)); - } - } - } } // if previous node success , post node submit @@ -1459,7 +1359,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } } log.info("The dependTasks of task all success, currentTaskCode: {}, dependTaskCodes: {}", - taskCode, Arrays.toString(completeTaskSet.toArray())); + taskCode, Arrays.toString(indirectDepCodeList.toArray())); return DependResult.SUCCESS; } @@ -1907,14 +1807,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { continue; } } - // init varPool only this task is the first time running if (task.isFirstRun()) { - // get pre task ,get all the task varPool to this task - // Do not use dag.getPreviousNodes because of the dag may be miss the upstream node - String preTasks = workflowExecuteContext.getWorkflowGraph() - .getTaskNodeByCode(task.getTaskCode()).getPreTasks(); - Set preTaskList = new HashSet<>(JSONUtils.toList(preTasks, Long.class)); - getPreVarPool(task, preTaskList); + initializeTaskInstanceVarPool(task); } DependResult dependResult = getDependResultForTask(task); if (DependResult.SUCCESS == dependResult) { @@ -2095,38 +1989,14 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { taskInstanceDao.updateById(taskInstance); } - Set removeSet = new HashSet<>(); - for (TaskInstance taskInstance : removeTaskInstances) { - String taskVarPool = taskInstance.getVarPool(); - if (StringUtils.isNotEmpty(taskVarPool)) { - List properties = JSONUtils.toList(taskVarPool, Property.class); - List keys = properties.stream() - .filter(property -> property.getDirect().equals(Direct.OUT)) - .map(property -> String.format("%s_%s", property.getProp(), property.getType())) - .collect(Collectors.toList()); - removeSet.addAll(keys); - } - } - - // remove varPool data and update process instance - // TODO: we can remove this snippet if : we get varPool from pre taskInstance instead of process instance when - // task can not get pre task from incomplete dag - List processProperties = JSONUtils.toList(workflowInstance.getVarPool(), Property.class); - processProperties = processProperties.stream() - .filter(property -> !(property.getDirect().equals(Direct.IN) - && removeSet.contains(String.format("%s_%s", property.getProp(), property.getType())))) - .collect(Collectors.toList()); - - workflowInstance.setVarPool(JSONUtils.toJsonString(processProperties)); + workflowInstance.setVarPool( + VarPoolUtils.subtractVarPoolJson(workflowInstance.getVarPool(), + removeTaskInstances.stream().map(TaskInstance::getVarPool).collect(Collectors.toList()))); processInstanceDao.updateById(workflowInstance); - // remove task instance from taskInstanceMap, completeTaskSet, validTaskMap, errorTaskMap - // completeTaskSet remove dependency taskInstanceMap, so the sort can't change - completeTaskSet.removeIf(taskCode -> { - Optional existTaskInstanceOptional = getTaskInstance(taskCode); - return existTaskInstanceOptional - .filter(taskInstance -> dag.containsNode(taskInstance.getTaskCode())).isPresent(); - }); + // remove task instance from taskInstanceMap,taskCodeInstanceMap , completeTaskSet, validTaskMap, errorTaskMap + completeTaskSet.removeIf(dag::containsNode); + taskCodeInstanceMap.entrySet().removeIf(entity -> dag.containsNode(entity.getValue().getTaskCode())); taskInstanceMap.entrySet().removeIf(entry -> dag.containsNode(entry.getValue().getTaskCode())); validTaskMap.entrySet().removeIf(entry -> dag.containsNode(entry.getKey())); errorTaskMap.entrySet().removeIf(entry -> dag.containsNode(entry.getKey())); @@ -2158,25 +2028,4 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } } - private void mergeTaskInstanceVarPool(TaskInstance taskInstance) { - String taskVarPoolJson = taskInstance.getVarPool(); - if (StringUtils.isEmpty(taskVarPoolJson)) { - return; - } - ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); - String processVarPoolJson = workflowInstance.getVarPool(); - if (StringUtils.isEmpty(processVarPoolJson)) { - workflowInstance.setVarPool(taskVarPoolJson); - return; - } - List processVarPool = new ArrayList<>(JSONUtils.toList(processVarPoolJson, Property.class)); - List taskVarPool = JSONUtils.toList(taskVarPoolJson, Property.class); - Set newProcessVarPoolKeys = taskVarPool.stream().map(Property::getProp).collect(Collectors.toSet()); - processVarPool = processVarPool.stream().filter(property -> !newProcessVarPoolKeys.contains(property.getProp())) - .collect(Collectors.toList()); - - processVarPool.addAll(taskVarPool); - - workflowInstance.setVarPool(JSONUtils.toJsonString(processVarPool)); - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java index 8fa2e2926d..72073359d3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java @@ -17,14 +17,13 @@ package org.apache.dolphinscheduler.server.master.runner.operator; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.GlobalTaskDispatchWaitingQueue; -import java.util.concurrent.TimeUnit; - import lombok.extern.slf4j.Slf4j; @Slf4j @@ -43,16 +42,17 @@ public abstract class BaseTaskExecuteRunnableDispatchOperator implements TaskExe @Override public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { - long remainTime = taskExecuteRunnable.getDelay(TimeUnit.SECONDS); TaskInstance taskInstance = taskExecuteRunnable.getTaskInstance(); - if (remainTime > 0) { + long remainTimeMills = + DateUtils.getRemainTime(taskInstance.getFirstSubmitTime(), taskInstance.getDelayTime() * 60L) * 1_000; + if (remainTimeMills > 0) { taskInstance.setState(TaskExecutionStatus.DELAY_EXECUTION); taskInstanceDao.updateById(taskInstance); - log.info("Current taskInstance: {} is choose delay execution, delay time: {}/min, remainTime: {}/s", + log.info("Current taskInstance: {} is choose delay execution, delay time: {}/min, remainTime: {}/ms", taskInstance.getName(), taskInstance.getDelayTime(), - remainTime); + remainTimeMills); } - globalTaskDispatchWaitingQueue.submitTaskExecuteRunnable(taskExecuteRunnable); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnableWithDelay(taskExecuteRunnable, remainTimeMills); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntry.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntry.java new file mode 100644 index 0000000000..da6a750261 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntry.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.queue; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Objects; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + +import lombok.Getter; + +import org.jetbrains.annotations.NotNull; + +public class DelayEntry> implements Delayed { + + private final long delayTimeMills; + + private final long triggerTimeMills; + + @Getter + private final V data; + + public DelayEntry(long delayTimeMills, V data) { + this.delayTimeMills = delayTimeMills; + this.triggerTimeMills = System.currentTimeMillis() + delayTimeMills; + this.data = checkNotNull(data, "data is null"); + } + + @Override + public long getDelay(@NotNull TimeUnit unit) { + long remainTimeMills = triggerTimeMills - System.currentTimeMillis(); + if (TimeUnit.MILLISECONDS.equals(unit)) { + return remainTimeMills; + } + return unit.convert(remainTimeMills, TimeUnit.MILLISECONDS); + } + + @Override + public int compareTo(@NotNull Delayed o) { + DelayEntry other = (DelayEntry) o; + int delayTimeMillsCompareResult = Long.compare(delayTimeMills, other.delayTimeMills); + if (delayTimeMillsCompareResult != 0) { + return delayTimeMillsCompareResult; + } + + if (data == null || other.data == null) { + return 0; + } + return data.compareTo(other.data); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + DelayEntry that = (DelayEntry) o; + return delayTimeMills == that.delayTimeMills && Objects.equals(data, that.data); + } + + @Override + public int hashCode() { + return Objects.hash(delayTimeMills, data); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/PriorityDelayQueue.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/PriorityDelayQueue.java new file mode 100644 index 0000000000..8ed4869625 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/queue/PriorityDelayQueue.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.queue; + +import java.util.concurrent.DelayQueue; + +import lombok.SneakyThrows; + +public class PriorityDelayQueue { + + private final DelayQueue queue = new DelayQueue<>(); + + public void add(V v) { + queue.put(v); + } + + @SneakyThrows + public V take() { + return queue.take(); + } + + public int size() { + return queue.size(); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java index e360a8857e..2401562f15 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java @@ -65,6 +65,7 @@ public class DynamicCommandUtils { command.setProcessInstancePriority(processInstance.getProcessInstancePriority()); command.setWorkerGroup(processInstance.getWorkerGroup()); command.setDryRun(processInstance.getDryRun()); + command.setTenantCode(processInstance.getTenantCode()); return command; } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java new file mode 100644 index 0000000000..1e5a77edb3 --- /dev/null +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.server.master.config.MasterConfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MasterRpcServerTest { + + private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig()); + + @Test + void testStart() { + Assertions.assertDoesNotThrow(masterRpcServer::start); + } + + @Test + void testClose() { + Assertions.assertDoesNotThrow(masterRpcServer::close); + } +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueTest.java new file mode 100644 index 0000000000..843456b98f --- /dev/null +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner; + +import static com.google.common.truth.Truth.assertThat; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; + +import org.apache.commons.lang3.time.DateUtils; + +import java.time.Duration; +import java.util.Date; +import java.util.concurrent.CompletableFuture; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class GlobalTaskDispatchWaitingQueueTest { + + private GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue; + + @BeforeEach + public void setUp() { + globalTaskDispatchWaitingQueue = new GlobalTaskDispatchWaitingQueue(); + } + + @Test + void submitTaskExecuteRunnable() { + TaskExecuteRunnable taskExecuteRunnable = createTaskExecuteRunnable(); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable); + Awaitility.await() + .atMost(Duration.ofSeconds(1)) + .untilAsserted( + () -> Assertions.assertNotNull(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable())); + } + + @Test + void testSubmitTaskExecuteRunnableWithDelay() { + TaskExecuteRunnable taskExecuteRunnable = createTaskExecuteRunnable(); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnableWithDelay(taskExecuteRunnable, 3_000L); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable); + + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable()).isNotNull(); + Awaitility.await() + .atLeast(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(4)) + .untilAsserted( + () -> Assertions.assertNotNull(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable())); + } + + @Test + void takeTaskExecuteRunnable_NoElementShouldBlock() { + CompletableFuture completableFuture = + CompletableFuture.runAsync(() -> globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable()); + assertThrowsExactly(ConditionTimeoutException.class, + () -> await() + .atLeast(Duration.ofSeconds(2)) + .timeout(Duration.ofSeconds(3)) + .until(completableFuture::isDone)); + } + + @Test + void takeTaskExecuteRunnable_withDifferentTaskInstancePriority() { + TaskExecuteRunnable taskExecuteRunnable1 = createTaskExecuteRunnable(); + taskExecuteRunnable1.getTaskInstance().setId(1); + taskExecuteRunnable1.getTaskInstance().setTaskInstancePriority(Priority.MEDIUM); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable1); + + TaskExecuteRunnable taskExecuteRunnable2 = createTaskExecuteRunnable(); + taskExecuteRunnable2.getTaskInstance().setId(2); + taskExecuteRunnable2.getTaskInstance().setTaskInstancePriority(Priority.HIGH); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable2); + + TaskExecuteRunnable taskExecuteRunnable3 = createTaskExecuteRunnable(); + taskExecuteRunnable3.getTaskInstance().setId(3); + taskExecuteRunnable3.getTaskInstance().setTaskInstancePriority(Priority.LOW); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable3); + + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(2); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(1); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(3); + } + + @Test + void takeTaskExecuteRunnable_withDifferentTaskGroupPriority() { + TaskExecuteRunnable taskExecuteRunnable1 = createTaskExecuteRunnable(); + taskExecuteRunnable1.getTaskInstance().setId(1); + taskExecuteRunnable1.getTaskInstance().setTaskGroupPriority(Priority.MEDIUM.getCode()); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable1); + + TaskExecuteRunnable taskExecuteRunnable2 = createTaskExecuteRunnable(); + taskExecuteRunnable2.getTaskInstance().setId(2); + taskExecuteRunnable2.getTaskInstance().setTaskGroupPriority(Priority.HIGH.getCode()); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable2); + + TaskExecuteRunnable taskExecuteRunnable3 = createTaskExecuteRunnable(); + taskExecuteRunnable3.getTaskInstance().setId(3); + taskExecuteRunnable3.getTaskInstance().setTaskGroupPriority(Priority.LOW.getCode()); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable3); + + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(3); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(1); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(2); + } + + @Test + void takeTaskExecuteRunnable_withDifferentSubmitTime() { + Date now = new Date(); + + TaskExecuteRunnable taskExecuteRunnable1 = createTaskExecuteRunnable(); + taskExecuteRunnable1.getTaskInstance().setId(1); + taskExecuteRunnable1.getTaskInstance().setFirstSubmitTime(now); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable1); + + TaskExecuteRunnable taskExecuteRunnable2 = createTaskExecuteRunnable(); + taskExecuteRunnable2.getTaskInstance().setId(2); + taskExecuteRunnable2.getTaskInstance().setFirstSubmitTime(DateUtils.addMinutes(now, 1)); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable2); + + TaskExecuteRunnable taskExecuteRunnable3 = createTaskExecuteRunnable(); + taskExecuteRunnable3.getTaskInstance().setId(3); + taskExecuteRunnable3.getTaskInstance().setFirstSubmitTime(DateUtils.addMinutes(now, -1)); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable3); + + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(3); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(1); + assertThat(globalTaskDispatchWaitingQueue.takeTaskExecuteRunnable().getTaskInstance().getId()) + .isEqualTo(2); + } + + @Test + void getWaitingDispatchTaskNumber() { + Assertions.assertEquals(0, globalTaskDispatchWaitingQueue.getWaitingDispatchTaskNumber()); + TaskExecuteRunnable taskExecuteRunnable = createTaskExecuteRunnable(); + globalTaskDispatchWaitingQueue.dispatchTaskExecuteRunnable(taskExecuteRunnable); + Assertions.assertEquals(1, globalTaskDispatchWaitingQueue.getWaitingDispatchTaskNumber()); + } + + private TaskExecuteRunnable createTaskExecuteRunnable() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setProcessInstancePriority(Priority.MEDIUM); + + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setTaskInstancePriority(Priority.MEDIUM); + taskInstance.setFirstSubmitTime(new Date()); + + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + + return new DefaultTaskExecuteRunnable(processInstance, taskInstance, taskExecutionContext, + new TaskExecuteRunnableOperatorManager()); + } +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java index c08fb206f4..409f1b7691 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java @@ -71,6 +71,7 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; @ExtendWith(MockitoExtension.class) @@ -106,6 +107,8 @@ public class WorkflowExecuteRunnableTest { private TaskGroupCoordinator taskGroupCoordinator; + private WorkflowExecuteContext workflowExecuteContext; + @BeforeEach public void init() throws Exception { applicationContext = Mockito.mock(ApplicationContext.class); @@ -134,7 +137,7 @@ public class WorkflowExecuteRunnableTest { stateWheelExecuteThread = Mockito.mock(StateWheelExecuteThread.class); curingGlobalParamsService = Mockito.mock(CuringParamsService.class); ProcessAlertManager processAlertManager = Mockito.mock(ProcessAlertManager.class); - WorkflowExecuteContext workflowExecuteContext = Mockito.mock(WorkflowExecuteContext.class); + workflowExecuteContext = Mockito.mock(WorkflowExecuteContext.class); Mockito.when(workflowExecuteContext.getWorkflowInstance()).thenReturn(processInstance); IWorkflowGraph workflowGraph = Mockito.mock(IWorkflowGraph.class); Mockito.when(workflowExecuteContext.getWorkflowGraph()).thenReturn(workflowGraph); @@ -209,11 +212,13 @@ public class WorkflowExecuteRunnableTest { } @Test - public void testGetPreVarPool() { + public void testInitializeTaskInstanceVarPool() { try { - Set preTaskName = new HashSet<>(); - preTaskName.add(1L); - preTaskName.add(2L); + IWorkflowGraph workflowGraph = Mockito.mock(IWorkflowGraph.class); + Mockito.when(workflowExecuteContext.getWorkflowGraph()).thenReturn(workflowGraph); + TaskNode taskNode = Mockito.mock(TaskNode.class); + Mockito.when(workflowGraph.getTaskNodeByCode(Mockito.anyLong())).thenReturn(taskNode); + Mockito.when(taskNode.getPreTasks()).thenReturn(JSONUtils.toJsonString(Lists.newArrayList(1L, 2L))); TaskInstance taskInstance = new TaskInstance(); @@ -255,7 +260,7 @@ public class WorkflowExecuteRunnableTest { taskCodeInstanceMapField.setAccessible(true); taskCodeInstanceMapField.set(workflowExecuteThread, taskCodeInstanceMap); - workflowExecuteThread.getPreVarPool(taskInstance, preTaskName); + workflowExecuteThread.initializeTaskInstanceVarPool(taskInstance); Assertions.assertNotNull(taskInstance.getVarPool()); taskInstance2.setVarPool("[{\"direct\":\"OUT\",\"prop\":\"test1\",\"type\":\"VARCHAR\",\"value\":\"2\"}]"); @@ -266,7 +271,7 @@ public class WorkflowExecuteRunnableTest { taskInstanceMapField.setAccessible(true); taskInstanceMapField.set(workflowExecuteThread, taskInstanceMap); - workflowExecuteThread.getPreVarPool(taskInstance, preTaskName); + workflowExecuteThread.initializeTaskInstanceVarPool(taskInstance); Assertions.assertNotNull(taskInstance.getVarPool()); } catch (Exception e) { Assertions.fail(); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityDelayTaskExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityDelayTaskExecuteRunnableTest.java deleted file mode 100644 index 778884e066..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityDelayTaskExecuteRunnableTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.runner.execute; - -import org.apache.dolphinscheduler.common.enums.Priority; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.PriorityDelayTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class PriorityDelayTaskExecuteRunnableTest { - - @Test - public void testCompareTo() { - TaskExecuteRunnableOperatorManager taskOperatorManager = new TaskExecuteRunnableOperatorManager(); - - ProcessInstance workflowInstance = new ProcessInstance(); - workflowInstance.setId(1); - workflowInstance.setProcessInstancePriority(Priority.HIGH); - - TaskInstance t1 = new TaskInstance(); - t1.setId(1); - t1.setTaskInstancePriority(Priority.HIGH); - - TaskInstance t2 = new TaskInstance(); - t2.setId(1); - t2.setTaskInstancePriority(Priority.HIGH); - - TaskExecutionContext context1 = new TaskExecutionContext(); - TaskExecutionContext context2 = new TaskExecutionContext(); - PriorityDelayTaskExecuteRunnable p1 = - new DefaultTaskExecuteRunnable(workflowInstance, t1, context1, taskOperatorManager); - PriorityDelayTaskExecuteRunnable p2 = - new DefaultTaskExecuteRunnable(workflowInstance, t2, context2, taskOperatorManager); - - Assertions.assertEquals(0, p1.compareTo(p2)); - - // the higher priority, the higher priority - t2.setTaskInstancePriority(Priority.MEDIUM); - Assertions.assertTrue(p1.compareTo(p2) < 0); - - // the smaller dispatch fail times, the higher priority - context1.setDispatchFailTimes(1); - Assertions.assertTrue(p1.compareTo(p2) > 0); - } - -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntryTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntryTest.java new file mode 100644 index 0000000000..0e3ab5da36 --- /dev/null +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/queue/DelayEntryTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.queue; + +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import com.google.common.truth.Truth; + +class DelayEntryTest { + + @Test + void getDelay() { + DelayEntry delayEntry = new DelayEntry<>(1_000L, "Item"); + Truth.assertThat(delayEntry.getDelay(TimeUnit.NANOSECONDS)) + .isWithin(100) + .of(TimeUnit.NANOSECONDS.convert(1_000L, TimeUnit.MILLISECONDS)); + } +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java index d238869f41..d9b9c82e66 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtilsTest.java @@ -54,6 +54,7 @@ class DynamicCommandUtilsTest { processInstance.setWarningGroupId(1); processInstance.setProcessInstancePriority(null); // update this processInstance.setWorkerGroup("worker"); + processInstance.setTenantCode("unit-root"); processInstance.setDryRun(0); } @@ -73,6 +74,7 @@ class DynamicCommandUtilsTest { Assertions.assertEquals(processInstance.getProcessInstancePriority(), command.getProcessInstancePriority()); Assertions.assertEquals(processInstance.getWorkerGroup(), command.getWorkerGroup()); Assertions.assertEquals(processInstance.getDryRun(), command.getDryRun()); + Assertions.assertEquals(processInstance.getTenantCode(), command.getTenantCode()); } @Test diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java index 1a3e4ab1e2..496983118f 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.microbench.rpc; -import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; @@ -46,18 +45,17 @@ import org.openjdk.jmh.infra.Blackhole; @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) public class RpcBenchMarkTest extends AbstractBaseBenchmark { - private NettyRemotingServer nettyRemotingServer; + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; private IService iService; @Setup public void before() { - nettyRemotingServer = new NettyRemotingServer( - NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build()); - nettyRemotingServer.start(); - SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery = - new SpringServerMethodInvokerDiscovery(nettyRemotingServer); + NettyServerConfig nettyServerConfig = + NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build(); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); springServerMethodInvokerDiscovery.postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); + springServerMethodInvokerDiscovery.start(); iService = SingletonJdkDynamicRpcClientProxyFactory.getProxyClient("localhost:12345", IService.class); } @@ -72,6 +70,6 @@ public class RpcBenchMarkTest extends AbstractBaseBenchmark { @TearDown public void after() { - nettyRemotingServer.close(); + springServerMethodInvokerDiscovery.close(); } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java index 8bdb8b9021..f90ef1ea32 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java @@ -26,10 +26,20 @@ import java.util.Collection; import lombok.NonNull; /** - * Registry + * The SPI interface for registry center, each registry plugin should implement this interface. */ public interface Registry extends Closeable { + /** + * Start the registry, once started, the registry will connect to the registry center. + */ + void start(); + + /** + * Whether the registry is connected + * + * @return true if connected, false otherwise. + */ boolean isConnected(); /** @@ -40,7 +50,13 @@ public interface Registry extends Closeable { */ void connectUntilTimeout(@NonNull Duration timeout) throws RegistryException; - boolean subscribe(String path, SubscribeListener listener); + /** + * Subscribe the path, when the path has expose {@link Event}, the listener will be triggered. + * + * @param path the path to subscribe + * @param listener the listener to be triggered + */ + void subscribe(String path, SubscribeListener listener); /** * Remove the path from the subscribe list. @@ -53,35 +69,34 @@ public interface Registry extends Closeable { void addConnectionStateListener(ConnectionListener listener); /** - * @return the value + * Get the value of the key, if key not exist will throw {@link RegistryException} */ - String get(String key); + String get(String key) throws RegistryException; /** + * Put the key-value pair into the registry * - * @param key - * @param value + * @param key the key, cannot be null + * @param value the value, cannot be null * @param deleteOnDisconnect if true, when the connection state is disconnected, the key will be deleted */ void put(String key, String value, boolean deleteOnDisconnect); /** - * This function will delete the keys whose prefix is {@param key} - * @param key the prefix of deleted key - * @throws if the key not exists, there is a registryException + * Delete the key from the registry */ void delete(String key); /** - * @return {@code true} if key exists. - * E.g: registry contains the following keys:[/test/test1/test2,] - * if the key: /test - * Return: test1 + * Return the children of the key */ Collection children(String key); /** - * @return if key exists,return true + * Check if the key exists + * + * @param key the key to check + * @return true if the key exists */ boolean exists(String key); @@ -90,6 +105,11 @@ public interface Registry extends Closeable { */ boolean acquireLock(String key); + /** + * Acquire the lock of the prefix {@param key}, if acquire in the given timeout return true, else return false. + */ + boolean acquireLock(String key, long timeout); + /** * Release the lock of the prefix {@param key} */ diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java index a1f3bb02b7..cbc2db6a63 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java @@ -37,5 +37,4 @@ public enum RegistryNodeType { private final String name; private final String registryPath; - } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java new file mode 100644 index 0000000000..5dca5552b3 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.Registry; + +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import com.google.common.collect.Lists; + +@Slf4j +public abstract class AbstractHAServer implements HAServer { + + private final Registry registry; + + private final String serverPath; + + private ServerStatus serverStatus; + + private final List serverStatusChangeListeners; + + public AbstractHAServer(Registry registry, String serverPath) { + this.registry = registry; + this.serverPath = serverPath; + this.serverStatus = ServerStatus.STAND_BY; + this.serverStatusChangeListeners = Lists.newArrayList(new DefaultServerStatusChangeListener()); + } + + @Override + public void start() { + registry.subscribe(serverPath, event -> { + if (Event.Type.REMOVE.equals(event.type())) { + if (isActive() && !participateElection()) { + statusChange(ServerStatus.STAND_BY); + } + } + }); + ScheduledExecutorService electionSelectionThread = + ThreadUtils.newSingleDaemonScheduledExecutorService("election-selection-thread"); + electionSelectionThread.schedule(() -> { + if (isActive()) { + return; + } + if (participateElection()) { + statusChange(ServerStatus.ACTIVE); + } + }, 10, TimeUnit.SECONDS); + } + + @Override + public boolean isActive() { + return ServerStatus.ACTIVE.equals(getServerStatus()); + } + + @Override + public boolean participateElection() { + return registry.acquireLock(serverPath, 3_000); + } + + @Override + public void addServerStatusChangeListener(ServerStatusChangeListener listener) { + serverStatusChangeListeners.add(listener); + } + + @Override + public ServerStatus getServerStatus() { + return serverStatus; + } + + @Override + public void shutdown() { + if (isActive()) { + registry.releaseLock(serverPath); + } + } + + private void statusChange(ServerStatus targetStatus) { + synchronized (this) { + ServerStatus originStatus = serverStatus; + serverStatus = targetStatus; + serverStatusChangeListeners.forEach(listener -> listener.change(originStatus, serverStatus)); + } + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java new file mode 100644 index 0000000000..f2e332ea20 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractServerStatusChangeListener.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractServerStatusChangeListener implements ServerStatusChangeListener { + + @Override + public void change(HAServer.ServerStatus originStatus, HAServer.ServerStatus currentStatus) { + log.info("The status change from {} to {}.", originStatus, currentStatus); + if (originStatus == HAServer.ServerStatus.ACTIVE) { + if (currentStatus == HAServer.ServerStatus.STAND_BY) { + changeToStandBy(); + } + } else if (originStatus == HAServer.ServerStatus.STAND_BY) { + if (currentStatus == HAServer.ServerStatus.ACTIVE) { + changeToActive(); + } + } + } + + public abstract void changeToActive(); + + public abstract void changeToStandBy(); +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java new file mode 100644 index 0000000000..d2acbcb516 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/DefaultServerStatusChangeListener.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DefaultServerStatusChangeListener extends AbstractServerStatusChangeListener { + + @Override + public void changeToActive() { + log.info("The status is active now."); + } + + @Override + public void changeToStandBy() { + log.info("The status is standby now."); + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java new file mode 100644 index 0000000000..6a79e6eb84 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/HAServer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +/** + * Interface for HA server, used to select a active server from multiple servers. + * In HA mode, there are multiple servers, only one server is active, others are standby. + */ +public interface HAServer { + + /** + * Start the server. + */ + void start(); + + /** + * Judge whether the server is active. + * + * @return true if the current server is active. + */ + boolean isActive(); + + /** + * Participate in the election of active server, this method will block until the server is active. + */ + boolean participateElection(); + + /** + * Add a listener to listen to the status change of the server. + * + * @param listener listener to add. + */ + void addServerStatusChangeListener(ServerStatusChangeListener listener); + + /** + * Get the status of the server. + * + * @return the status of the server. + */ + ServerStatus getServerStatus(); + + /** + * Shutdown the server, release resources. + */ + void shutdown(); + + enum ServerStatus { + ACTIVE, + STAND_BY, + ; + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java new file mode 100644 index 0000000000..af109228e2 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/ServerStatusChangeListener.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api.ha; + +public interface ServerStatusChangeListener { + + void change(HAServer.ServerStatus originStatus, HAServer.ServerStatus currentStatus); + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml index b084db1ccf..0f5c4d1494 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml @@ -31,6 +31,15 @@ org.apache.dolphinscheduler dolphinscheduler-registry-api + + + org.apache.dolphinscheduler + dolphinscheduler-registry-it + ${project.version} + test-jar + test + + io.etcd jetcd-core @@ -49,18 +58,22 @@ + + + io.netty + netty-all + + io.etcd jetcd-test test + - io.netty - netty-all - - - org.slf4j - slf4j-api + org.springframework.boot + spring-boot-starter-test + test diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 6833a6607b..80279775ff 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -34,6 +34,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.net.ssl.SSLException; @@ -69,6 +71,7 @@ import io.netty.handler.ssl.SslContext; @Slf4j public class EtcdRegistry implements Registry { + private final EtcdRegistryProperties etcdRegistryProperties; private final Client client; private EtcdConnectionStateListener etcdConnectionStateListener; @@ -81,9 +84,8 @@ public class EtcdRegistry implements Registry { private final Map watcherMap = new ConcurrentHashMap<>(); - private static final long TIME_TO_LIVE_SECONDS = 30L; - public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException { + this.etcdRegistryProperties = registryProperties; ClientBuilder clientBuilder = Client.builder() .endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints()))) .namespace(byteSequence(registryProperties.getNamespace())) @@ -127,6 +129,11 @@ public class EtcdRegistry implements Registry { } + @Override + public void start() { + // The start has been set in the constructor + } + @Override public boolean isConnected() { return client.getKVClient().get(byteSequence("/")).join() != null; @@ -143,7 +150,7 @@ public class EtcdRegistry implements Registry { * @return if subcribe Returns true if no exception was thrown */ @Override - public boolean subscribe(String path, SubscribeListener listener) { + public void subscribe(String path, SubscribeListener listener) { try { ByteSequence watchKey = byteSequence(path); WatchOption watchOption = @@ -157,7 +164,6 @@ public class EtcdRegistry implements Registry { } catch (Exception e) { throw new RegistryException("Failed to subscribe listener for key: " + path, e); } - return true; } /** @@ -191,7 +197,7 @@ public class EtcdRegistry implements Registry { } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("etcd get data error", e); - } catch (ExecutionException e) { + } catch (Exception e) { throw new RegistryException("etcd get data error, key = " + key, e); } } @@ -204,7 +210,8 @@ public class EtcdRegistry implements Registry { try { if (deleteOnDisconnect) { // keep the key by lease, if disconnected, the lease will expire and the key will delete - long leaseId = etcdKeepAliveLeaseManager.getOrCreateKeepAliveLease(key, TIME_TO_LIVE_SECONDS); + long leaseId = etcdKeepAliveLeaseManager.getOrCreateKeepAliveLease(key, + etcdRegistryProperties.getTtl().get(ChronoUnit.SECONDS)); PutOption putOption = PutOption.newBuilder().withLeaseId(leaseId).build(); client.getKVClient().put(byteSequence(key), byteSequence(value), putOption).get(); } else { @@ -287,26 +294,67 @@ public class EtcdRegistry implements Registry { */ @Override public boolean acquireLock(String key) { + Map leaseIdMap = threadLocalLockMap.get(); + if (null == leaseIdMap) { + leaseIdMap = new HashMap<>(); + threadLocalLockMap.set(leaseIdMap); + } + if (leaseIdMap.containsKey(key)) { + return true; + } + Lock lockClient = client.getLockClient(); Lease leaseClient = client.getLeaseClient(); // get the lock with a lease try { - long leaseId = leaseClient.grant(TIME_TO_LIVE_SECONDS).get().getID(); + long leaseId = leaseClient.grant(etcdRegistryProperties.getTtl().get(ChronoUnit.SECONDS)).get().getID(); // keep the lease client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { })); lockClient.lock(byteSequence(key), leaseId).get(); // save the leaseId for release Lock - if (null == threadLocalLockMap.get()) { - threadLocalLockMap.set(new HashMap<>()); - } - threadLocalLockMap.get().put(key, leaseId); + leaseIdMap.put(key, leaseId); return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("etcd get lock error", e); - } catch (ExecutionException e) { + } catch (Exception e) { + throw new RegistryException("etcd get lock error, lockKey: " + key, e); + } + } + + @Override + public boolean acquireLock(String key, long timeout) { + Map leaseIdMap = threadLocalLockMap.get(); + if (null == leaseIdMap) { + leaseIdMap = new HashMap<>(); + threadLocalLockMap.set(leaseIdMap); + } + if (leaseIdMap.containsKey(key)) { + return true; + } + + Lock lockClient = client.getLockClient(); + Lease leaseClient = client.getLeaseClient(); + // get the lock with a lease + try { + long leaseId = leaseClient.grant(etcdRegistryProperties.getTtl().get(ChronoUnit.SECONDS)).get().getID(); + // keep the lease + lockClient.lock(byteSequence(key), leaseId).get(timeout, TimeUnit.MILLISECONDS); + client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { + })); + + // save the leaseId for release Lock + leaseIdMap.put(key, leaseId); + return true; + } catch (TimeoutException timeoutException) { + log.debug("Acquire lock: {} in {}/ms timeout", key, timeout); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RegistryException("etcd get lock error", e); + } catch (Exception e) { throw new RegistryException("etcd get lock error, lockKey: " + key, e); } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index babb6dea76..b748c2a0fe 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -33,6 +33,8 @@ public class EtcdRegistryProperties { private String namespace = "dolphinscheduler"; private Duration connectionTimeout = Duration.ofSeconds(9); + private Duration ttl = Duration.ofSeconds(30); + // auth private String user; private String password; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java index 70593e4bad..84acbae8f3 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdKeepAliveLeaseManagerTest.java @@ -36,13 +36,14 @@ class EtcdKeepAliveLeaseManagerTest { static Client client; static EtcdKeepAliveLeaseManager etcdKeepAliveLeaseManager; + @BeforeAll public static void before() throws Exception { server = EtcdClusterExtension.builder() .withNodes(1) .withImage("ibmcom/etcd:3.2.24") .build(); - server.restart(); + server.cluster().start(); client = Client.builder().endpoints(server.clientEndpoints()).build(); @@ -65,8 +66,9 @@ class EtcdKeepAliveLeaseManagerTest { @AfterAll public static void after() throws IOException { - try (EtcdCluster closeServer = server.cluster()) { - client.close(); + try ( + EtcdCluster closeServer = server.cluster(); + Client closedClient = client) { } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTest.java deleted file mode 100644 index b99bab98ad..0000000000 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.registry.etcd; - -import org.apache.dolphinscheduler.registry.api.Event; -import org.apache.dolphinscheduler.registry.api.SubscribeListener; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.etcd.jetcd.test.EtcdClusterExtension; - -public class EtcdRegistryTest { - - private static final Logger logger = LoggerFactory.getLogger(EtcdRegistryTest.class); - - public static EtcdRegistry registry; - - @BeforeAll - public static void before() throws Exception { - EtcdClusterExtension server = EtcdClusterExtension.builder() - .withNodes(1) - .withImage("ibmcom/etcd:3.2.24") - .build(); - EtcdRegistryProperties properties = new EtcdRegistryProperties(); - server.restart(); - properties.setEndpoints(String.valueOf(server.clientEndpoints().get(0))); - registry = new EtcdRegistry(properties); - registry.put("/sub", "sub", false); - } - - @Test - public void persistTest() { - registry.put("/nodes/m1", "", false); - registry.put("/nodes/m2", "", false); - Assertions.assertEquals(Arrays.asList("m1", "m2"), registry.children("/nodes")); - Assertions.assertTrue(registry.exists("/nodes/m1")); - registry.delete("/nodes/m2"); - Assertions.assertFalse(registry.exists("/nodes/m2")); - registry.delete("/nodes"); - Assertions.assertFalse(registry.exists("/nodes/m1")); - } - - @Test - public void lockTest() { - CountDownLatch preCountDownLatch = new CountDownLatch(1); - CountDownLatch allCountDownLatch = new CountDownLatch(2); - List testData = new ArrayList<>(); - new Thread(() -> { - registry.acquireLock("/lock"); - preCountDownLatch.countDown(); - logger.info(Thread.currentThread().getName() - + " :I got the lock, but I don't want to work. I want to rest for a while"); - try { - Thread.sleep(1000); - logger.info(Thread.currentThread().getName() + " :I'm going to start working"); - testData.add("thread1"); - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - logger.info(Thread.currentThread().getName() + " :I have finished my work, now I release the lock"); - registry.releaseLock("/lock"); - allCountDownLatch.countDown(); - } - }).start(); - try { - preCountDownLatch.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - new Thread(() -> { - try { - logger.info(Thread.currentThread().getName() + " :I am trying to acquire the lock"); - registry.acquireLock("/lock"); - logger.info(Thread.currentThread().getName() + " :I got the lock and I started working"); - - testData.add("thread2"); - } finally { - registry.releaseLock("/lock"); - allCountDownLatch.countDown(); - } - - }).start(); - try { - allCountDownLatch.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - Assertions.assertEquals(testData, Arrays.asList("thread1", "thread2")); - } - - @Test - public void subscribeTest() { - boolean status = registry.subscribe("/sub", new TestListener()); - // The following add and delete operations are used for debugging - registry.put("/sub/m1", "tt", false); - registry.put("/sub/m2", "tt", false); - registry.delete("/sub/m2"); - registry.delete("/sub"); - Assertions.assertTrue(status); - - } - - static class TestListener implements SubscribeListener { - - @Override - public void notify(Event event) { - logger.info("I'm test listener"); - } - } - - @AfterAll - public static void after() throws IOException { - registry.close(); - } -} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java new file mode 100644 index 0000000000..1e751c1862 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.etcd; + +import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase; + +import java.net.URI; +import java.util.stream.Collectors; + +import lombok.SneakyThrows; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; + +import io.etcd.jetcd.launcher.EtcdCluster; +import io.etcd.jetcd.test.EtcdClusterExtension; + +@SpringBootTest(classes = EtcdRegistryProperties.class) +@SpringBootApplication(scanBasePackageClasses = EtcdRegistryProperties.class) +public class EtcdRegistryTestCase extends RegistryTestCase { + + @Autowired + private EtcdRegistryProperties etcdRegistryProperties; + + private static EtcdCluster etcdCluster; + + @SneakyThrows + @BeforeAll + public static void setUpTestingServer() { + etcdCluster = EtcdClusterExtension.builder() + .withNodes(1) + .withImage("ibmcom/etcd:3.2.24") + .build() + .cluster(); + etcdCluster.start(); + System.setProperty("registry.endpoints", + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","))); + } + + @SneakyThrows + @Override + public EtcdRegistry createRegistry() { + return new EtcdRegistry(etcdRegistryProperties); + } + + @SneakyThrows + @AfterAll + public static void tearDownTestingServer() { + try (EtcdCluster cluster = etcdCluster) { + } + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/application.yaml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/application.yaml new file mode 100644 index 0000000000..083d38511c --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/application.yaml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +registry: + type: etcd + ttl: 2s diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/logback.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/logback.xml new file mode 100644 index 0000000000..6f211959c5 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/pom.xml new file mode 100644 index 0000000000..7f4b97d3ef --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/pom.xml @@ -0,0 +1,60 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-registry-plugins + dev-SNAPSHOT + + + dolphinscheduler-registry-it + + + + org.apache.dolphinscheduler + dolphinscheduler-registry-api + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + false + + + + + test-jar + + + + + + + + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java new file mode 100644 index 0000000000..8fbd6bc5c0 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.dolphinscheduler.registry.api.ConnectionState; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.RegistryException; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import lombok.SneakyThrows; + +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.google.common.truth.Truth; + +public abstract class RegistryTestCase { + + protected R registry; + + @BeforeEach + public void setupRegistry() { + registry = createRegistry(); + } + + @SneakyThrows + @AfterEach + public void tearDownRegistry() { + try (R registry = this.registry) { + } + } + + @Test + public void testIsConnected() { + registry.start(); + Truth.assertThat(registry.isConnected()).isTrue(); + } + + @Test + public void testConnectUntilTimeout() { + registry.start(); + await().atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> registry.connectUntilTimeout(Duration.ofSeconds(3))); + + } + + @SneakyThrows + @Test + public void testSubscribe() { + registry.start(); + + final AtomicBoolean subscribeAdded = new AtomicBoolean(false); + final AtomicBoolean subscribeRemoved = new AtomicBoolean(false); + final AtomicBoolean subscribeUpdated = new AtomicBoolean(false); + + SubscribeListener subscribeListener = event -> { + System.out.println("Receive event: " + event); + if (event.type() == Event.Type.ADD) { + subscribeAdded.compareAndSet(false, true); + } + if (event.type() == Event.Type.REMOVE) { + subscribeRemoved.compareAndSet(false, true); + } + if (event.type() == Event.Type.UPDATE) { + subscribeUpdated.compareAndSet(false, true); + } + }; + String key = "/nodes/master" + System.nanoTime(); + registry.subscribe(key, subscribeListener); + registry.put(key, String.valueOf(System.nanoTime()), true); + // Sleep 3 seconds here since in mysql jdbc registry + // If multiple event occurs in a refresh time, only the last event will be triggered + Thread.sleep(3000); + registry.put(key, String.valueOf(System.nanoTime()), true); + Thread.sleep(3000); + registry.delete(key); + + await().atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> { + Assertions.assertTrue(subscribeAdded.get()); + Assertions.assertTrue(subscribeUpdated.get()); + Assertions.assertTrue(subscribeRemoved.get()); + }); + } + + @SneakyThrows + @Test + public void testUnsubscribe() { + registry.start(); + + final AtomicBoolean subscribeAdded = new AtomicBoolean(false); + final AtomicBoolean subscribeRemoved = new AtomicBoolean(false); + final AtomicBoolean subscribeUpdated = new AtomicBoolean(false); + + SubscribeListener subscribeListener = event -> { + if (event.type() == Event.Type.ADD) { + subscribeAdded.compareAndSet(false, true); + } + if (event.type() == Event.Type.REMOVE) { + subscribeRemoved.compareAndSet(false, true); + } + if (event.type() == Event.Type.UPDATE) { + subscribeUpdated.compareAndSet(false, true); + } + }; + String key = "/nodes/master" + System.nanoTime(); + String value = "127.0.0.1:8080"; + registry.subscribe(key, subscribeListener); + registry.unsubscribe(key); + registry.put(key, value, true); + registry.put(key, value, true); + registry.delete(key); + + Thread.sleep(2000); + Assertions.assertFalse(subscribeAdded.get()); + Assertions.assertFalse(subscribeRemoved.get()); + Assertions.assertFalse(subscribeUpdated.get()); + + } + + @SneakyThrows + @Test + public void testAddConnectionStateListener() { + + AtomicReference connectionState = new AtomicReference<>(); + registry.addConnectionStateListener(connectionState::set); + + Truth.assertThat(connectionState.get()).isNull(); + registry.start(); + + await().atMost(Duration.ofSeconds(2)) + .until(() -> ConnectionState.CONNECTED == connectionState.get()); + + } + + @Test + public void testGet() { + registry.start(); + String key = "/nodes/master" + System.nanoTime(); + String value = "127.0.0.1:8080"; + assertThrows(RegistryException.class, () -> registry.get(key)); + registry.put(key, value, true); + Truth.assertThat(registry.get(key)).isEqualTo(value); + } + + @Test + public void testPut() { + registry.start(); + String key = "/nodes/master" + System.nanoTime(); + String value = "127.0.0.1:8080"; + registry.put(key, value, true); + Truth.assertThat(registry.get(key)).isEqualTo(value); + + // Update the value + registry.put(key, "123", true); + Truth.assertThat(registry.get(key)).isEqualTo("123"); + } + + @Test + public void testDelete() { + registry.start(); + String key = "/nodes/master" + System.nanoTime(); + String value = "127.0.0.1:8080"; + // Delete a non-existent key + registry.delete(key); + + registry.put(key, value, true); + Truth.assertThat(registry.get(key)).isEqualTo(value); + registry.delete(key); + Truth.assertThat(registry.exists(key)).isFalse(); + + } + + @Test + public void testChildren() { + registry.start(); + String master1 = "/nodes/children/127.0.0.1:8080"; + String master2 = "/nodes/children/127.0.0.2:8080"; + String value = "123"; + registry.put(master1, value, true); + registry.put(master2, value, true); + Truth.assertThat(registry.children("/nodes/children")) + .containsAtLeastElementsIn(Lists.newArrayList("127.0.0.1:8080", "127.0.0.2:8080")); + } + + @Test + public void testExists() { + registry.start(); + String key = "/nodes/master" + System.nanoTime(); + String value = "123"; + Truth.assertThat(registry.exists(key)).isFalse(); + registry.put(key, value, true); + Truth.assertThat(registry.exists(key)).isTrue(); + + } + + @SneakyThrows + @Test + public void testAcquireLock() { + registry.start(); + String lockKey = "/lock" + System.nanoTime(); + + // 1. Acquire the lock at the main thread + Truth.assertThat(registry.acquireLock(lockKey)).isTrue(); + // Acquire the lock at the main thread again + // It should acquire success + Truth.assertThat(registry.acquireLock(lockKey)).isTrue(); + + // Acquire the lock at another thread + // It should acquire failed + CompletableFuture acquireResult = CompletableFuture.supplyAsync(() -> registry.acquireLock(lockKey)); + assertThrows(TimeoutException.class, () -> acquireResult.get(3000, TimeUnit.MILLISECONDS)); + + } + + @SneakyThrows + @Test + public void testAcquireLock_withTimeout() { + registry.start(); + String lockKey = "/lock" + System.nanoTime(); + // 1. Acquire the lock in the main thread + Truth.assertThat(registry.acquireLock(lockKey, 3000)).isTrue(); + + // Acquire the lock in the main thread + // It should acquire success + Truth.assertThat(registry.acquireLock(lockKey, 3000)).isTrue(); + + // Acquire the lock at another thread + // It should acquire failed + CompletableFuture acquireResult = + CompletableFuture.supplyAsync(() -> registry.acquireLock(lockKey, 3000)); + Truth.assertThat(acquireResult.get()).isFalse(); + + } + + @SneakyThrows + @Test + public void testReleaseLock() { + registry.start(); + String lockKey = "/lock" + System.nanoTime(); + // 1. Acquire the lock in the main thread + Truth.assertThat(registry.acquireLock(lockKey, 3000)).isTrue(); + + // Acquire the lock at another thread + // It should acquire failed + CompletableFuture acquireResult = + CompletableFuture.supplyAsync(() -> registry.acquireLock(lockKey, 3000)); + Truth.assertThat(acquireResult.get()).isFalse(); + + // 2. Release the lock in the main thread + Truth.assertThat(registry.releaseLock(lockKey)).isTrue(); + + // Acquire the lock at another thread + // It should acquire success + acquireResult = CompletableFuture.supplyAsync(() -> registry.acquireLock(lockKey, 3000)); + Truth.assertThat(acquireResult.get()).isTrue(); + } + + public abstract R createRegistry(); + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml index d4285edfbd..aa592b9da4 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml @@ -72,6 +72,31 @@ + + + org.apache.dolphinscheduler + dolphinscheduler-registry-it + ${project.version} + test-jar + test + + + + org.testcontainers + mysql + + + + org.testcontainers + postgresql + + + + org.springframework.boot + spring-boot-starter-test + test + + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/EphemeralDateManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/EphemeralDateManager.java similarity index 93% rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/EphemeralDateManager.java rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/EphemeralDateManager.java index 64915e8ca8..7c601b91a1 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/EphemeralDateManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/EphemeralDateManager.java @@ -15,12 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.registry.jdbc.task; +package org.apache.dolphinscheduler.plugin.registry.jdbc; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcOperator; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryProperties; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; @@ -42,7 +40,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; * This thread is used to check the connect state to jdbc. */ @Slf4j -public class EphemeralDateManager implements AutoCloseable { +class EphemeralDateManager implements AutoCloseable { private ConnectionState connectionState; private final JdbcOperator jdbcOperator; @@ -51,7 +49,7 @@ public class EphemeralDateManager implements AutoCloseable { private final Set ephemeralDateIds = Collections.synchronizedSet(new HashSet<>()); private final ScheduledExecutorService scheduledExecutorService; - public EphemeralDateManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { + EphemeralDateManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { this.registryProperties = registryProperties; this.jdbcOperator = checkNotNull(jdbcOperator); this.scheduledExecutorService = Executors.newScheduledThreadPool( @@ -151,7 +149,7 @@ public class EphemeralDateManager implements AutoCloseable { } } - private void updateEphemeralDateTerm() throws SQLException { + private void updateEphemeralDateTerm() { if (!jdbcOperator.updateEphemeralDataTerm(ephemeralDateIds)) { log.warn("Update jdbc registry ephemeral data: {} term error", ephemeralDateIds); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcOperator.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcOperator.java index a56d609da7..95f58a4a20 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcOperator.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcOperator.java @@ -29,26 +29,25 @@ import org.apache.commons.lang3.StringUtils; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.util.Collection; +import java.util.Date; import java.util.List; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; +import org.springframework.dao.DuplicateKeyException; -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") -public class JdbcOperator { +public final class JdbcOperator { - @Autowired - private JdbcRegistryDataMapper jdbcRegistryDataMapper; - @Autowired - private JdbcRegistryLockMapper jdbcRegistryLockMapper; + private final JdbcRegistryDataMapper jdbcRegistryDataMapper; + private final JdbcRegistryLockMapper jdbcRegistryLockMapper; private final long expireTimeWindow; - public JdbcOperator(JdbcRegistryProperties registryProperties) { + JdbcOperator(JdbcRegistryProperties registryProperties, + JdbcRegistryDataMapper jdbcRegistryDataMapper, + JdbcRegistryLockMapper jdbcRegistryLockMapper) { this.expireTimeWindow = registryProperties.getTermExpireTimes() * registryProperties.getTermRefreshInterval().toMillis(); + this.jdbcRegistryDataMapper = jdbcRegistryDataMapper; + this.jdbcRegistryLockMapper = jdbcRegistryLockMapper; } public void healthCheck() { @@ -62,17 +61,21 @@ public class JdbcOperator { public Long insertOrUpdateEphemeralData(String key, String value) throws SQLException { JdbcRegistryData jdbcRegistryData = jdbcRegistryDataMapper.selectByKey(key); if (jdbcRegistryData != null) { - long id = jdbcRegistryData.getId(); - if (jdbcRegistryDataMapper.updateDataAndTermById(id, value, System.currentTimeMillis()) <= 0) { + jdbcRegistryData.setDataValue(value); + jdbcRegistryData.setLastUpdateTime(new Date()); + jdbcRegistryData.setLastTerm(System.currentTimeMillis()); + if (jdbcRegistryDataMapper.updateById(jdbcRegistryData) <= 0) { throw new SQLException(String.format("update registry value failed, key: %s, value: %s", key, value)); } - return id; + return jdbcRegistryData.getId(); } jdbcRegistryData = JdbcRegistryData.builder() .dataKey(key) .dataValue(value) .dataType(DataType.EPHEMERAL.getTypeValue()) .lastTerm(System.currentTimeMillis()) + .lastUpdateTime(new Date()) + .createTime(new Date()) .build(); jdbcRegistryDataMapper.insert(jdbcRegistryData); return jdbcRegistryData.getId(); @@ -81,17 +84,21 @@ public class JdbcOperator { public long insertOrUpdatePersistentData(String key, String value) throws SQLException { JdbcRegistryData jdbcRegistryData = jdbcRegistryDataMapper.selectByKey(key); if (jdbcRegistryData != null) { - long id = jdbcRegistryData.getId(); - if (jdbcRegistryDataMapper.updateDataAndTermById(id, value, System.currentTimeMillis()) <= 0) { + jdbcRegistryData.setDataValue(value); + jdbcRegistryData.setLastUpdateTime(new Date()); + jdbcRegistryData.setLastTerm(System.currentTimeMillis()); + if (jdbcRegistryDataMapper.updateById(jdbcRegistryData) <= 0) { throw new SQLException(String.format("update registry value failed, key: %s, value: %s", key, value)); } - return id; + return jdbcRegistryData.getId(); } jdbcRegistryData = JdbcRegistryData.builder() .dataKey(key) .dataValue(value) .dataType(DataType.PERSISTENT.getTypeValue()) .lastTerm(System.currentTimeMillis()) + .lastUpdateTime(new Date()) + .createTime(new Date()) .build(); jdbcRegistryDataMapper.insert(jdbcRegistryData); return jdbcRegistryData.getId(); @@ -127,7 +134,7 @@ public class JdbcOperator { .collect(Collectors.toList()); } - public boolean existKey(String key) throws SQLException { + public boolean existKey(String key) { JdbcRegistryData jdbcRegistryData = jdbcRegistryDataMapper.selectByKey(key); return jdbcRegistryData != null; } @@ -136,24 +143,25 @@ public class JdbcOperator { * Try to acquire the target Lock, if cannot acquire, return null. */ @SuppressWarnings("checkstyle:IllegalCatch") - public JdbcRegistryLock tryToAcquireLock(String key) throws SQLException { + public JdbcRegistryLock tryToAcquireLock(String key) { JdbcRegistryLock jdbcRegistryLock = JdbcRegistryLock.builder() .lockKey(key) - .lockOwner(JdbcRegistryConstant.LOCK_OWNER) + .lockOwner(LockUtils.getLockOwner()) .lastTerm(System.currentTimeMillis()) + .lastUpdateTime(new Date()) .build(); try { jdbcRegistryLockMapper.insert(jdbcRegistryLock); return jdbcRegistryLock; } catch (Exception e) { - if (e instanceof SQLIntegrityConstraintViolationException) { + if (e instanceof SQLIntegrityConstraintViolationException || e instanceof DuplicateKeyException) { return null; } throw e; } } - public JdbcRegistryLock getLockById(long lockId) throws SQLException { + public JdbcRegistryLock getLockById(long lockId) { return jdbcRegistryLockMapper.selectById(lockId); } @@ -161,7 +169,7 @@ public class JdbcOperator { return jdbcRegistryLockMapper.deleteById(lockId) > 0; } - public boolean updateEphemeralDataTerm(Collection ephemeralDateIds) throws SQLException { + public boolean updateEphemeralDataTerm(Collection ephemeralDateIds) { if (CollectionUtils.isEmpty(ephemeralDateIds)) { return true; } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java index f3cbcfbc3b..2b7993c87b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java @@ -17,9 +17,7 @@ package org.apache.dolphinscheduler.plugin.registry.jdbc; -import org.apache.dolphinscheduler.plugin.registry.jdbc.task.EphemeralDateManager; -import org.apache.dolphinscheduler.plugin.registry.jdbc.task.RegistryLockManager; -import org.apache.dolphinscheduler.plugin.registry.jdbc.task.SubscribeDataManager; +import org.apache.dolphinscheduler.plugin.registry.jdbc.model.JdbcRegistryData; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.registry.api.Registry; @@ -30,31 +28,24 @@ import java.sql.SQLException; import java.time.Duration; import java.util.Collection; -import javax.annotation.PostConstruct; - import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on mysql database to * store the DolphinScheduler master/worker's metadata and do the server registry/unRegistry. */ -@Component -@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "jdbc") @Slf4j -public class JdbcRegistry implements Registry { +public final class JdbcRegistry implements Registry { private final JdbcRegistryProperties jdbcRegistryProperties; private final EphemeralDateManager ephemeralDateManager; private final SubscribeDataManager subscribeDataManager; private final RegistryLockManager registryLockManager; - private JdbcOperator jdbcOperator; + private final JdbcOperator jdbcOperator; - public JdbcRegistry(JdbcRegistryProperties jdbcRegistryProperties, - JdbcOperator jdbcOperator) { + JdbcRegistry(JdbcRegistryProperties jdbcRegistryProperties, + JdbcOperator jdbcOperator) { this.jdbcOperator = jdbcOperator; jdbcOperator.clearExpireLock(); jdbcOperator.clearExpireEphemeralDate(); @@ -65,7 +56,7 @@ public class JdbcRegistry implements Registry { log.info("Initialize Jdbc Registry..."); } - @PostConstruct + @Override public void start() { log.info("Starting Jdbc Registry..."); // start a jdbc connect check @@ -103,10 +94,9 @@ public class JdbcRegistry implements Registry { } @Override - public boolean subscribe(String path, SubscribeListener listener) { + public void subscribe(String path, SubscribeListener listener) { // new a schedule thread to query the path, if the path subscribeDataManager.addListener(path, listener); - return true; } @Override @@ -122,8 +112,18 @@ public class JdbcRegistry implements Registry { @Override public String get(String key) { - // get the key value - return subscribeDataManager.getData(key); + try { + // get the key value + JdbcRegistryData data = jdbcOperator.getData(key); + if (data == null) { + throw new RegistryException("key: " + key + " not exist"); + } + return data.getDataValue(); + } catch (RegistryException registryException) { + throw registryException; + } catch (Exception e) { + throw new RegistryException(String.format("Get key: %s error", key), e); + } } @Override @@ -179,6 +179,17 @@ public class JdbcRegistry implements Registry { } } + @Override + public boolean acquireLock(String key, long timeout) { + try { + return registryLockManager.acquireLock(key, timeout); + } catch (RegistryException e) { + throw e; + } catch (Exception e) { + throw new RegistryException(String.format("Acquire lock: %s error", key), e); + } + } + @Override public boolean releaseLock(String key) { registryLockManager.releaseLock(key); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java index f21ce0d67c..603a476322 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java @@ -49,6 +49,18 @@ public class JdbcRegistryAutoConfiguration { log.info("Load JdbcRegistryAutoConfiguration"); } + @Bean + public JdbcOperator jdbcOperator(JdbcRegistryProperties jdbcRegistryProperties, + JdbcRegistryDataMapper jdbcRegistryDataMapper, + JdbcRegistryLockMapper jdbcRegistryLockMapper) { + return new JdbcOperator(jdbcRegistryProperties, jdbcRegistryDataMapper, jdbcRegistryLockMapper); + } + + @Bean + public JdbcRegistry jdbcRegistry(JdbcRegistryProperties jdbcRegistryProperties, JdbcOperator jdbcOperator) { + return new JdbcRegistry(jdbcRegistryProperties, jdbcOperator); + } + @Bean @ConditionalOnMissingBean public SqlSessionFactory sqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties) throws Exception { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConstant.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConstant.java index 4a016f4d2e..84496afb80 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConstant.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConstant.java @@ -17,15 +17,11 @@ package org.apache.dolphinscheduler.plugin.registry.jdbc; -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; - import lombok.experimental.UtilityClass; @UtilityClass -public final class JdbcRegistryConstant { +final class JdbcRegistryConstant { public static final long LOCK_ACQUIRE_INTERVAL = 1_000; - public static final String LOCK_OWNER = NetUtils.getHost() + "_" + OSUtils.getProcessID(); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/LockUtils.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/LockUtils.java new file mode 100644 index 0000000000..f70f0afa5b --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/LockUtils.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.jdbc; + +import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class LockUtils { + + private static final String LOCK_OWNER_PREFIX = NetUtils.getHost() + "_" + OSUtils.getProcessID() + "_"; + + public static String getLockOwner() { + return LOCK_OWNER_PREFIX + Thread.currentThread().getName(); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/RegistryLockManager.java similarity index 65% rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/RegistryLockManager.java index 46ccd15ec0..6c519685ff 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/RegistryLockManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/RegistryLockManager.java @@ -15,12 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.registry.jdbc.task; +package org.apache.dolphinscheduler.plugin.registry.jdbc; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcOperator; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryConstant; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryProperties; import org.apache.dolphinscheduler.plugin.registry.jdbc.model.JdbcRegistryLock; import org.apache.dolphinscheduler.registry.api.RegistryException; @@ -40,14 +37,15 @@ import lombok.extern.slf4j.Slf4j; import com.google.common.util.concurrent.ThreadFactoryBuilder; @Slf4j -public class RegistryLockManager implements AutoCloseable { +class RegistryLockManager implements AutoCloseable { private final JdbcOperator jdbcOperator; private final JdbcRegistryProperties registryProperties; + // lock owner -> lock private final Map lockHoldMap; private final ScheduledExecutorService lockTermUpdateThreadPool; - public RegistryLockManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { + RegistryLockManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { this.registryProperties = registryProperties; this.jdbcOperator = jdbcOperator; this.lockHoldMap = new ConcurrentHashMap<>(); @@ -67,20 +65,49 @@ public class RegistryLockManager implements AutoCloseable { * Acquire the lock, if cannot get the lock will await. */ public void acquireLock(String lockKey) throws RegistryException { - // maybe we can use the computeIf absent - lockHoldMap.computeIfAbsent(lockKey, key -> { - JdbcRegistryLock jdbcRegistryLock; - try { - while ((jdbcRegistryLock = jdbcOperator.tryToAcquireLock(lockKey)) == null) { - log.debug("Acquire the lock {} failed try again", key); - // acquire failed, wait and try again - ThreadUtils.sleep(JdbcRegistryConstant.LOCK_ACQUIRE_INTERVAL); + try { + while (true) { + JdbcRegistryLock jdbcRegistryLock = lockHoldMap.get(lockKey); + if (jdbcRegistryLock != null && LockUtils.getLockOwner().equals(jdbcRegistryLock.getLockOwner())) { + return; } - } catch (SQLException e) { - throw new RegistryException("Acquire the lock error", e); + jdbcRegistryLock = jdbcOperator.tryToAcquireLock(lockKey); + if (jdbcRegistryLock != null) { + lockHoldMap.put(lockKey, jdbcRegistryLock); + return; + } + log.debug("Acquire the lock {} failed try again", lockKey); + // acquire failed, wait and try again + ThreadUtils.sleep(JdbcRegistryConstant.LOCK_ACQUIRE_INTERVAL); } - return jdbcRegistryLock; - }); + } catch (Exception ex) { + throw new RegistryException("Acquire the lock: " + lockKey + " error", ex); + } + } + + /** + * Acquire the lock, if cannot get the lock will await. + */ + public boolean acquireLock(String lockKey, long timeout) throws RegistryException { + long startTime = System.currentTimeMillis(); + try { + while (System.currentTimeMillis() - startTime < timeout) { + JdbcRegistryLock jdbcRegistryLock = lockHoldMap.get(lockKey); + if (jdbcRegistryLock != null && LockUtils.getLockOwner().equals(jdbcRegistryLock.getLockOwner())) { + return true; + } + jdbcRegistryLock = jdbcOperator.tryToAcquireLock(lockKey); + if (jdbcRegistryLock != null) { + lockHoldMap.put(lockKey, jdbcRegistryLock); + return true; + } + log.debug("Acquire the lock {} failed try again", lockKey); + ThreadUtils.sleep(JdbcRegistryConstant.LOCK_ACQUIRE_INTERVAL); + } + } catch (Exception e) { + throw new RegistryException("Acquire the lock: " + lockKey + " error", e); + } + return false; } public void releaseLock(String lockKey) { @@ -91,6 +118,7 @@ public class RegistryLockManager implements AutoCloseable { jdbcOperator.releaseLock(jdbcRegistryLock.getId()); lockHoldMap.remove(lockKey); } catch (SQLException e) { + lockHoldMap.remove(lockKey); throw new RegistryException(String.format("Release lock: %s error", lockKey), e); } } @@ -125,7 +153,6 @@ public class RegistryLockManager implements AutoCloseable { if (!jdbcOperator.updateLockTerm(lockIds)) { log.warn("Update the lock: {} term failed.", lockIds); } - jdbcOperator.clearExpireLock(); } catch (Exception e) { log.error("Update lock term error", e); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/SubscribeDataManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/SubscribeDataManager.java similarity index 91% rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/SubscribeDataManager.java rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/SubscribeDataManager.java index 4718b053f4..e86dc4b155 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/task/SubscribeDataManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/SubscribeDataManager.java @@ -15,10 +15,8 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.registry.jdbc.task; +package org.apache.dolphinscheduler.plugin.registry.jdbc; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcOperator; -import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryProperties; import org.apache.dolphinscheduler.plugin.registry.jdbc.model.JdbcRegistryData; import org.apache.dolphinscheduler.registry.api.Event; import org.apache.dolphinscheduler.registry.api.SubscribeListener; @@ -42,7 +40,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; * Used to refresh if the subscribe path has been changed. */ @Slf4j -public class SubscribeDataManager implements AutoCloseable { +class SubscribeDataManager implements AutoCloseable { private final JdbcOperator jdbcOperator; private final JdbcRegistryProperties registryProperties; @@ -50,7 +48,7 @@ public class SubscribeDataManager implements AutoCloseable { private final ScheduledExecutorService dataSubscribeCheckThreadPool; private final Map jdbcRegistryDataMap = new ConcurrentHashMap<>(); - public SubscribeDataManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { + SubscribeDataManager(JdbcRegistryProperties registryProperties, JdbcOperator jdbcOperator) { this.registryProperties = registryProperties; this.jdbcOperator = jdbcOperator; this.dataSubscribeCheckThreadPool = Executors.newScheduledThreadPool( @@ -75,12 +73,8 @@ public class SubscribeDataManager implements AutoCloseable { dataSubScribeMap.remove(path); } - public String getData(String path) { - JdbcRegistryData jdbcRegistryData = jdbcRegistryDataMap.get(path); - if (jdbcRegistryData == null) { - return null; - } - return jdbcRegistryData.getDataValue(); + public JdbcRegistryData getData(String path) { + return jdbcRegistryDataMap.get(path); } @Override @@ -107,6 +101,7 @@ public class SubscribeDataManager implements AutoCloseable { List addedData = new ArrayList<>(); List deletedData = new ArrayList<>(); List updatedData = new ArrayList<>(); + for (Map.Entry entry : currentJdbcDataMap.entrySet()) { JdbcRegistryData newData = entry.getValue(); JdbcRegistryData oldData = jdbcRegistryDataMap.get(entry.getKey()); @@ -118,6 +113,7 @@ public class SubscribeDataManager implements AutoCloseable { } } } + for (Map.Entry entry : jdbcRegistryDataMap.entrySet()) { if (!currentJdbcDataMap.containsKey(entry.getKey())) { deletedData.add(entry.getValue()); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryDataMapper.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryDataMapper.java index 701f2e7310..e1d27bbf0b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryDataMapper.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryDataMapper.java @@ -40,9 +40,6 @@ public interface JdbcRegistryDataMapper extends BaseMapper { @Select("select * from t_ds_jdbc_registry_data where data_key like CONCAT (#{key}, '%')") List fuzzyQueryByKey(@Param("key") String key); - @Update("update t_ds_jdbc_registry_data set data_value = #{data}, last_term = #{term} where id = #{id}") - int updateDataAndTermById(@Param("id") long id, @Param("data") String data, @Param("term") long term); - @Delete("delete from t_ds_jdbc_registry_data where data_key = #{key}") void deleteByKey(@Param("key") String key); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryLockMapper.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryLockMapper.java index 2d11c90a24..0f529a8786 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryLockMapper.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryLockMapper.java @@ -38,7 +38,7 @@ public interface JdbcRegistryLockMapper extends BaseMapper { @Update({"