natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into dev

This commit is contained in:
yinxiaolog 2024-06-04 16:07:09 +08:00
parent 21846a6a4e 4f6e7159b8
commit 8f5aa0664d
469 changed files with 9294 additions and 3660 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.asf.yaml
View File

@ -46,7 +46,7 @@ github:
- E2E
- Docs
- Frontend Build
- "Mergeable: milestone-label-check"
# - "Mergeable: milestone-label-check"
required_pull_request_reviews:
dismiss_stale_reviews: true
required_approving_review_count: 2

62
.github/mergeable.yml vendored
View File

@ -1,62 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
version: 2
mergeable:
# we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
# otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
# GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
# which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
- when: pull_request.opened, pull_request.reopened, pull_request.synchronize
name: sync-sql-ddl
validate:
# Sql files must change synchronize
- do: dependent
files:
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
message: 'Sql files not change synchronize'
# Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
fail:
- do: comment
payload:
body: >
:warning: This PR do not change database DDL synchronize.
leave_old_comment: false
- do: labels
add: 'sql not sync'
# Remove labels 'sql not sync' if pass
pass:
- do: labels
delete: 'sql not sync'
- when: pull_request.*
name: milestone-label-check
validate:
- do: milestone
no_empty:
enabled: false # Cannot be empty when true.
message: 'Milestone is required and cannot be empty.'
- do: label
and:
- must_include:
regex: 'feature|bug|improvement|document|chore|revert'
message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
- must_include:
regex: 'ready-to-merge'
message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'

2
.github/workflows/backend.yml vendored
View File

@ -153,7 +153,7 @@ jobs:
fail-fast: false
matrix:
db: ["mysql", "postgresql"]
version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"]
version: ["3.1.9", "3.2.0"]
steps:
- name: Set up JDK 8
uses: actions/setup-java@v2

78
.github/workflows/mergeable.yml vendored Normal file
View File

@ -0,0 +1,78 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
#version: 2
on:
pull_request:
name: "Mergeable"
jobs:
result:
name: "Mergeable: milestone-label-check"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Status
run: |
echo "Temporary skipping this check"
#mergeable:
# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
# name: sync-sql-ddl
# validate:
# # Sql files must change synchronize
# - do: dependent
# files:
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
# message: 'Sql files not change synchronize'
# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
# fail:
# - do: comment
# payload:
# body: >
# :warning: This PR do not change database DDL synchronize.
# leave_old_comment: false
# - do: labels
# add: 'sql not sync'
# # Remove labels 'sql not sync' if pass
# pass:
# - do: labels
# delete: 'sql not sync'
#
# - when: pull_request.*
# name: milestone-label-check
# validate:
# - do: milestone
# no_empty:
# enabled: false # Cannot be empty when true.
# message: 'Milestone is required and cannot be empty.'
# - do: label
# and:
# - must_include:
# regex: 'feature|bug|improvement|document|chore|revert'
# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
# - must_include:
# regex: 'ready-to-merge'
# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'

16
.github/workflows/unit-test.yml vendored
View File

@ -76,7 +76,7 @@ jobs:
restore-keys: ${{ runner.os }}-maven-
- name: Run Unit tests
run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false
run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false
- name: Upload coverage report to codecov
run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
@ -99,23 +99,11 @@ jobs:
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
-Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/*
-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-DskipUT=true -DskipIT=true
-DskipUT=true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
- name: Collect logs
continue-on-error: true
run: |
mkdir -p ${LOG_DIR}
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
- name: Upload logs
uses: actions/upload-artifact@v2
continue-on-error: true
with:
name: unit-test-logs
path: ${LOG_DIR}
result:
name: Unit Test
runs-on: ubuntu-latest

13
deploy/kubernetes/dolphinscheduler/README.md
View File

@ -120,6 +120,12 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
| conf.common."alert.rpc.port" | int | `50052` | rpc port |
| conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | |
| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required |
| conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
| conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
| conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
@ -138,11 +144,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
| conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
| conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
| conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
| conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
| conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
@ -158,6 +159,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| conf.common."yarn.application.status.address" | string | `"http://ds1:%s/ws/v1/cluster/apps/%s"` | if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname |
| conf.common."yarn.job.history.status.address" | string | `"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"` | job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) |
| conf.common."yarn.resourcemanager.ha.rm.ids" | string | `"192.168.xx.xx,192.168.xx.xx"` | if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty |
| datasource.profile | string | `"postgresql"` | The profile of datasource |
| externalDatabase.database | string | `"dolphinscheduler"` | The database of external database |
| externalDatabase.driverClassName | string | `"org.postgresql.Driver"` | The driverClassName of external database |
| externalDatabase.enabled | bool | `false` | If exists external database, and set postgresql.enable value to false. external database will be used, otherwise Dolphinscheduler's internal database will be used. |
@ -306,7 +308,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. |
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. |
| worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. |
| worker.keda.advanced | object | `{}` | Specify HPA related options |
| worker.keda.cooldownPeriod | int | `30` | How many seconds KEDA will wait before scaling to zero. Note that HPA has a separate cooldown period for scale-downs |
| worker.keda.enabled | bool | `false` | Enable or disable the Keda component |

5
deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
View File

@ -51,7 +51,6 @@ Create a default common labels.
{{- define "dolphinscheduler.common.labels" -}}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/version: {{ .Chart.AppVersion }}
{{- end -}}
{{/*
@ -146,6 +145,10 @@ Create a database environment variables.
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
{{- if or .Values.mysql.enabled (eq .Values.externalDatabase.type "mysql") }}
- name: SPRING_PROFILES_ACTIVE
value: mysql
{{- end }}
- name: SPRING_DATASOURCE_URL
{{- if .Values.postgresql.enabled }}
value: jdbc:postgresql://{{ template "dolphinscheduler.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresqlDatabase }}?{{ .Values.postgresql.params }}

115
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml Normal file
View File

@ -0,0 +1,115 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.alert.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
profiles:
active: {{ .Values.datasource.profile }}
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
banner:
charset: UTF-8
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
server:
port: 50053
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
alert:
port: 50052
# Mark each alert of alert server if late after x milliseconds as failed.
# Define value is (0 = infinite), and alert server would be waiting alert result.
wait-timeout: 0
max-heartbeat-interval: 60s
query_alert_threshold: 100
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
metrics:
enabled: true
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
{{- end }}

258
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml Normal file
View File

@ -0,0 +1,258 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.api.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.api.labels" . | nindent 4 }}
data:
application.yaml: |
server:
port: 12345
servlet:
session:
timeout: 120m
context-path: /dolphinscheduler/
compression:
enabled: true
mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
jetty:
max-http-form-post-size: 5000000
accesslog:
enabled: true
custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
spring:
profiles:
active: {{ .Values.datasource.profile }}
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
quartz:
auto-startup: false
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
mvc:
pathmatch:
matching-strategy: ANT_PATH_MATCHER
static-path-pattern: /static/**
springdoc:
swagger-ui:
path: /swagger-ui.html
packages-to-scan: org.apache.dolphinscheduler.api
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 60s
connection-timeout: 15s
block-until-connected: 15s
digest: ~
api:
audit-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
traffic-control:
global-switch: false
max-global-qps-rate: 300
tenant-switch: false
default-tenant-qps-rate: 10
#customize-tenant-qps-rate:
# eg.
#tenant1: 11
#tenant2: 20
python-gateway:
# Weather enable python gateway server or not. The default value is false.
enabled: false
# Authentication token for connection from python api to python gateway server. Should be changed the default value
# when you deploy in public network.
auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
gateway-server-address: 0.0.0.0
# The port of Python gateway server start. Define which port you could connect to Python gateway server from
# Python API side.
gateway-server-port: 25333
# The address of Python callback client.
python-address: 127.0.0.1
# The port of Python callback client.
python-port: 25334
# Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
# and socket server would never close even though no requests accept
connect-timeout: 0
# Close each active connection of socket server if python program not active after x milliseconds. Define value is
# (0 = infinite), and socket server would never close even though no requests accept
read-timeout: 0
metrics:
enabled: true
security:
authentication:
# Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
type: PASSWORD
# IF you set type `LDAP`, below config will be effective
ldap:
# ldap server config
urls: ldap://ldap.forumsys.com:389/
base-dn: dc=example,dc=com
username: cn=read-only-admin,dc=example,dc=com
password: password
user:
# admin userId when you use LDAP login
admin: read-only-admin
identity-attribute: uid
email-attribute: mail
# action when ldap user is not exist (supported types: CREATE,DENY)
not-exist-action: CREATE
ssl:
enable: false
# jks file absolute path && password
trust-store: "/ldapkeystore.jks"
trust-store-password: "password"
casdoor:
user:
admin: ""
oauth2:
enable: false
provider:
github:
authorizationUri: ""
redirectUri: ""
clientId: ""
clientSecret: ""
tokenUri: ""
userInfoUri: ""
callbackUrl: ""
iconUri: ""
provider: github
google:
authorizationUri: ""
redirectUri: ""
clientId: ""
clientSecret: ""
tokenUri: ""
userInfoUri: ""
callbackUrl: ""
iconUri: ""
provider: google
casdoor:
# Your Casdoor server url
endpoint: ""
client-id: ""
client-secret: ""
# The certificate may be multi-line, you can use `|-` for ease
certificate: ""
# Your organization name added in Casdoor
organization-name: ""
# Your application name added in Casdoor
application-name: ""
# Doplhinscheduler login url
redirect-url: ""
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
{{- end }}

173
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml Normal file
View File

@ -0,0 +1,173 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.master.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
{{- include "dolphinscheduler.master.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
profiles:
active: {{ .Values.datasource.profile }}
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
quartz:
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.threadPool.threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.threadPool.makeThreadsDaemons: true
org.quartz.threadPool.threadCount: 25
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
master:
listen-port: 5678
# master fetch command num
fetch-command-num: 10
# master prepare execute thread number to limit handle commands in parallel
pre-exec-threads: 10
# master execute thread number to limit process instances in parallel
exec-threads: 100
# master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s.
dispatch-task-number: 3
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
host-selector: lower_weight
# master heartbeat interval
max-heartbeat-interval: 10s
# master commit task retry times
task-commit-retry-times: 5
# master commit task interval
task-commit-interval: 1s
state-wheel-interval: 5s
server-load-protection:
# If set true, will open master overload protection
enabled: true
# Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
max-system-cpu-usage-percentage-thresholds: 0.7
# Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
max-jvm-cpu-usage-percentage-thresholds: 0.7
# Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
max-system-memory-usage-percentage-thresholds: 0.7
# Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
max-disk-usage-percentage-thresholds: 0.7
# failover interval, the unit is minute
failover-interval: 10m
# kill yarn / k8s application when failover taskInstance, default true
kill-application-when-task-failover: true
registry-disconnect-strategy:
# The disconnect strategy: stop, waiting
strategy: waiting
# The max waiting time to reconnect to registry if you set the strategy to waiting
max-waiting-time: 100s
worker-group-refresh-interval: 10s
server:
port: 5679
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
metrics:
enabled: true
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
{{- end }}

108
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml Normal file
View File

@ -0,0 +1,108 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.worker.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-worker
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
autoconfigure:
exclude:
- org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
worker:
# worker listener port
listen-port: 1234
# worker execute thread number to limit task instances in parallel
exec-threads: 100
# worker heartbeat interval
max-heartbeat-interval: 10s
# worker host weight to dispatch tasks, default value 100
host-weight: 100
server-load-protection:
# If set true, will open worker overload protection
enabled: true
# Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
max-system-cpu-usage-percentage-thresholds: 0.7
# Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
max-jvm-cpu-usage-percentage-thresholds: 0.7
# Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
max-system-memory-usage-percentage-thresholds: 0.7
# Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
max-disk-usage-percentage-thresholds: 0.7
registry-disconnect-strategy:
# The disconnect strategy: stop, waiting
strategy: waiting
# The max waiting time to reconnect to registry if you set the strategy to waiting
max-waiting-time: 100s
task-execute-threads-full-policy: REJECT
tenant-config:
# tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
auto-create-tenant-enabled: true
# Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants.
distributed-tenant-enabled: false
# If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
default-tenant-enabled: false
server:
port: 1235
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
metrics:
enabled: true
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
View File

@ -37,7 +37,7 @@ data:
{{- range $key, $value := index .Values.conf "common" }}
{{- if and $.Values.minio.enabled }}
{{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
{{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
{{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
{{- end }}
{{ $key }}={{ $value }}
{{- end }}

6
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
View File

@ -115,6 +115,9 @@ spec:
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
- name: alert-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- if .Values.alert.persistentVolumeClaim.enabled }}
@ -126,4 +129,7 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
- name: alert-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- end }}

6
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

@ -121,6 +121,9 @@ spec:
mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
subPath: task-type-config.yaml
{{- end }}
- name: api-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
@ -136,6 +139,9 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
- name: api-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}

6
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

@ -109,6 +109,9 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-master
- name: master-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
@ -122,6 +125,9 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
- name: master-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-master
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
- name: config-volume
configMap:

6
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

@ -111,6 +111,9 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-worker-data
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
- name: worker-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
@ -139,6 +142,9 @@ spec:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
emptyDir: {}
{{- end }}
- name: worker-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-worker
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs

27
deploy/kubernetes/dolphinscheduler/values.yaml
View File

@ -49,6 +49,10 @@ image:
# -- tools image
tools: dolphinscheduler-tools
datasource:
# -- The profile of datasource
profile: postgresql
postgresql:
# -- If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL
enabled: true
@ -246,20 +250,25 @@ conf:
# -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path: /dolphinscheduler
# -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id: minioadmin
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
aws.credentials.provider.type: AWSStaticCredentialsProvider
# -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key: minioadmin
# -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
aws.s3.access.key.id: minioadmin
# -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region: ca-central-1
# -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
aws.s3.access.key.secret: minioadmin
# -- The AWS Region to use. if resource.storage.type=S3, This configuration is required
aws.s3.region: ca-central-1
# -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name: dolphinscheduler
aws.s3.bucket.name: dolphinscheduler
# -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint: http://minio:9000
aws.s3.endpoint: http://minio:9000
# -- alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id: <your-access-key-id>
@ -645,8 +654,6 @@ worker:
WORKER_HOST_WEIGHT: "100"
# -- tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: true
# -- Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT: false
# -- If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`;
DEFAULT_TENANT_ENABLED: false

24
docs/docs/en/architecture/configuration.md
View File

@ -165,7 +165,7 @@ The default configuration is as follows:
Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`.
### Zookeeper related configuration
### Registry Related configuration
DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions.
Configuration file location:
@ -191,16 +191,21 @@ The default configuration is as follows:
Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
For ETCD Registry, please see more details
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
For JDBC Registry, please see more details
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
### common.properties [hadoop、s3、yarn config properties]
Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
| Service | Configuration file |
|---------------|----------------------------------------|
| Master Server | `master-server/conf/common.properties` |
| Api Server | `api-server/conf/common.properties` |
| Worker Server | `worker-server/conf/common.properties` |
| Alert Server | `alert-server/conf/common.properties` |
| Service | Configuration file |
|---------------|-----------------------------------------------------------------------|
| Master Server | `master-server/conf/common.properties` |
| Api Server | `api-server/conf/common.properties`, `api-server/conf/aws.yaml` |
| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` |
| Alert Server | `alert-server/conf/common.properties` |
The default configuration is as follows:
@ -209,10 +214,6 @@ The default configuration is as follows:
| data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files |
| resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE |
| resource.upload.path | /dolphinscheduler | storage path of resource files |
| aws.access.key.id | minioadmin | access key id of S3 |
| aws.secret.access.key | minioadmin | secret access key of S3 |
| aws.region | us-east-1 | region of S3 |
| aws.s3.endpoint | http://minio:9000 | endpoint of S3 |
| hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS |
| fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory |
| hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission |
@ -327,7 +328,6 @@ Location: `worker-server/conf/application.yaml`
| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution |
| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants |
| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. |
### Alert Server related configuration

2
docs/docs/en/contribute/development-environment-setup.md
View File

@ -153,7 +153,7 @@ The browser access address [http://localhost:5173](http://localhost:5173) can lo
#### zookeeper
Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it.
Download [ZooKeeper](https://zookeeper.apache.org/releases.html), and extract it.
- Create directory `zkData` and `zkLog`
- Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper

2
docs/docs/en/guide/installation/cluster.md
View File

@ -14,7 +14,7 @@ Configure all the configurations refer to [pseudo-cluster deployment](pseudo-clu
### Modify Configuration
This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster .md), the following describes the parameters that must be modified:
This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster.md), the following describes the parameters that must be modified:
```shell
# ---------------------------------------------------------

2
docs/docs/en/guide/installation/pseudo-cluster.md
View File

@ -2,7 +2,7 @@
The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
## Preparation

2
docs/docs/en/guide/installation/standalone.md
View File

@ -36,7 +36,7 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch
### Start or Stop Server
The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
The script `./bin/dolphinscheduler-daemon.sh` can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
```shell
# Start Standalone Server

21
docs/docs/en/guide/monitor.md
View File

@ -16,6 +16,12 @@
![worker](../../../img/new_ui/dev/monitor/worker.png)
### Alert Server
- Mainly related to alert server information.
![alert-server](../../../img/new_ui/dev/monitor/alert-server.png)
### Database
- Mainly the health status of the DB.
@ -26,18 +32,17 @@
### Statistics
![statistics](../../../img/new_ui/dev/monitor/statistics.png)
![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png)
| **Parameter** | **Description** |
|----------------------------------------|----------------------------------------------------|
| Number of commands wait to be executed | Statistics of the `t_ds_command` table data. |
| The number of failed commands | Statistics of the `t_ds_error_command` table data. |
| Number of tasks wait to run | Count the data of `task_queue` in the ZooKeeper. |
| Number of tasks wait to be killed | Count the data of `task_kill` in the ZooKeeper. |
Shows the command list in the system. Data is from the `t_ds_command` table.
![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png)
Shows the failure command list in the system. Data is from the `t_ds_error_command` table.
### Audit Log
The audit log provides information about who accesses the system and the operations made to the system and record related
time, which strengthen the security of the system and maintenance.
![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg)
![audit-log](../../../img/new_ui/dev/monitor/audit-log.png)

2
docs/docs/en/guide/parameter/global.md
View File

@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case,
### Save the workflow and set global parameters
Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it.
Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it.
![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)

2
docs/docs/en/guide/parameter/project-parameter.md
View File

@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project.
### Define project-level parameters
On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below:
On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below:
![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)

2
docs/docs/en/guide/parameter/startup-parameter.md
View File

@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config
## Usage
Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters.
Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters.
## Example

168
docs/docs/en/guide/resource/configuration.md
View File

@ -2,8 +2,8 @@
- You could use `Resource Center` to upload text files, UDFs and other task-related files.
- You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html) etc.
- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resouce Center` without the need of an external `HDFS` system or `S3`.
- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resouce Center`. In this way, you could operate remote files as if on your local machines.
- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resource Center` without the need of an external `HDFS` system or `S3`.
- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resource Center`. In this way, you could operate remote files as if on your local machines.
## Use Local File System
@ -28,74 +28,37 @@ The configuration you may need to change:
## connect AWS S3
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following:
config the following fields
```properties
......
resource.storage.type=S3
......
resource.aws.access.key.id=aws_access_key_id
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=aws_secret_access_key
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=us-west-2
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name=dolphinscheduler
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint=
......
```
## Use HDFS or Remote Object Storage
```yaml
aws:
s3:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
bucket.name: <bucket.name>
endpoint: <endpoint>
After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`.
```
## connect OSS S3
if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
config the following fields
```properties
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
resource.storage.type=LOCAL
# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path=/tmp/dolphinscheduler
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=minioadmin
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=minioadmin
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=cn-north-1
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name=dolphinscheduler
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint=http://localhost:9000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id=<your-access-key-id>
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
@ -107,89 +70,24 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OSS
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
# alibaba cloud access key id, required if you set resource.storage.type=OBS
```
## connect OBS S3
if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
config the following fields
```properties
# access key id, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.id=<your-access-key-id>
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
# access key secret, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
# oss bucket name, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
resource.hdfs.root.user=hdfs
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
# whether to startup kerberos
hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path
java.security.krb5.conf.path=/opt/krb5.conf
# login user from keytab username
login.user.keytab.username=hdfs-mycluster@ESZ.COM
# login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab
# kerberos expire time, the unit is hour
kerberos.expire.time=2
# resourcemanager port, the default value is 8088 if not specified
resource.manager.httpaddress.port=8088
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
# datasource encryption enable
datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
# Network IP gets priority, default inner outer
# Whether hive SQL is executed in the same session
support.hive.oneSession=false
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=true
# network interface preferred like eth0, default: empty
#dolphin.scheduler.network.interface.preferred=
# network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default
# system env path
#dolphinscheduler.env.path=dolphinscheduler_env.sh
# development state
development.state=false
# rpc port
alert.rpc.port=50052
# set path of conda.sh
conda.path=/opt/anaconda3/etc/profile.d/conda.sh
# Task resource limit state
task.resource.limit.state=false
# way to collect applicationId: log(original regex match), aop
appId.collect: log
```
> **Note:**

2
docs/docs/en/guide/start/docker.md
View File

@ -128,7 +128,7 @@ and use `admin` and `dolphinscheduler123` as default username and password in th
![login](../../../../img/new_ui/dev/quick-start/login.png)
> Note: If you start the services by the way [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper), and
> strating with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running.
> starting with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running.
## Change Environment Variable

6
docs/docs/en/guide/task/datafactory.md
View File

@ -19,11 +19,11 @@ DolphinScheduler DataFactory functions:
### Application Permission Setting
First, visit the `Subcription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page.
![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
First, visit the `Subscription` page and choose `Access control (IAM)`, then click `Add role assignment` to the authorization page.
![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
After that, select `Contributor` role which satisfy functions calls in data factory. Then click `Members` page, and click `Select members`.
Search application name or application `Object ID` to assign `Contributor` role to application.
![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
## Configurations

17
docs/docs/en/guide/task/dms.md
View File

@ -73,14 +73,17 @@ Parameters of restarting the task by interface
## Environment to prepare
Some AWS configuration is required, modify a field in file `common.properties`
Some AWS configuration is required, modify a field in file `aws.yaml`
```yaml
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=<AWS REGION>
dms:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
endpoint: <endpoint>
```

2
docs/docs/en/guide/task/kubernetes.md
View File

@ -26,7 +26,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the
| Command | The container execution command (yaml-style array), for example: ["printenv"] |
| Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] |
| Custom label | The customized labels for k8s Job. |
| Node selector | The label selectors for running k8s pod. Different value in value set should be seperated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. |
| Node selector | The label selectors for running k8s pod. Different value in value set should be separated by comma, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. |
| Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. |
## Task Example

2
docs/docs/en/guide/task/mlflow.md
View File

@ -148,7 +148,7 @@ After this, you can visit the MLflow service (`http://localhost:5000`) page to v
### Preset Algorithm Repository Configuration
If you can't access github, you can modify the following fields in the `commom.properties` configuration file to replace the github address with an accessible address.
If you can't access github, you can modify the following fields in the `common.properties` configuration file to replace the github address with an accessible address.
```yaml
# mlflow task plugin preset repository

17
docs/docs/en/guide/task/sagemaker.md
View File

@ -35,14 +35,17 @@ The task plugin are shown as follows:
## Environment to prepare
Some AWS configuration is required, modify a field in file `common.properties`
Some AWS configuration is required, modify a field in file `aws.yaml`
```yaml
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=<AWS REGION>
sagemaker:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
endpoint: <endpoint>
```

8
docs/docs/zh/architecture/configuration.md
View File

@ -165,9 +165,9 @@ export DOLPHINSCHEDULER_OPTS="
DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名,
设置值即可。
## Zookeeper相关配置
## 注册中心相关配置
DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置：
DolphinScheduler默认使用Zookeeper进行集群管理、容错、事件监听等功能，配置文件位置：
|服务名称| 配置文件 |
|--|--|
|Master Server | `master-server/conf/application.yaml`|
@ -190,6 +190,9 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功
DolphinScheduler同样可以通过`bin/env/dolphinscheduler_env.sh`进行Zookeeper相关的配置。
如果使用etcd作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md)。
如果使用jdbc作为注册中心，详细请参考[链接](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md)。
## common.properties [hadoop、s3、yarn配置]
common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId收集相关的配置，配置文件位置：
@ -328,7 +331,6 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 |
| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 |
| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
## Alert Server相关配置

2
docs/docs/zh/contribute/development-environment-setup.md
View File

@ -148,7 +148,7 @@ pnpm run dev
#### zookeeper
下载 [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3)，解压
下载 [ZooKeeper](https://zookeeper.apache.org/releases.html)，解压
* 在 ZooKeeper 的目录下新建 zkData、zkLog文件夹
* 将 conf 目录下的 `zoo_sample.cfg` 文件，复制一份，重命名为 `zoo.cfg`，修改其中数据和日志的配置，如：

2
docs/docs/zh/contribute/join/contribute.md
View File

@ -32,7 +32,7 @@
如果你想实现某个 Feature 或者修复某个 Bug。请参考以下内容：
* 所有的 Bug 与新 Feature 建议使用 Issues Page 进行管理。
* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加的回复内容中。
* 如果想要开发实现某个 Feature 功能，请先回复该功能所关联的 Issue，表明你当前正在这个 Issue 上工作。 并在回复的时候为自己设置一个 **deadline**，并添加到回复内容中。
* 最好在核心贡献者找到一个导师（指导者），导师会在设计与功能实现上给予即时的反馈。
* 你应该新建一个分支来开始你的工作，分支的名字参考[参与贡献 Pull Request 需知](./pull-request.md)。比如，你想完成 feature 功能并提交了 Issue 111，那么你的 branch 名字应为 feature-111。 功能名称可与导师讨论后确定。
* 完成后，发送一个 Pull Request 到 dolphinscheduler，提交过程具体请参考下面《[提交代码流程](./submit-code.md)》。

21
docs/docs/zh/guide/monitor.md
View File

@ -16,6 +16,12 @@
![worker](../../../img/new_ui/dev/monitor/worker.png)
### Alert Server
- 主要是 alert server 的相关信息。
![alert-server](../../../img/new_ui/dev/monitor/alert-server.png)
### Database
- 主要是 DB 的健康状况
@ -26,15 +32,16 @@
### Statistics
![statistics](../../../img/new_ui/dev/monitor/statistics.png)
![Command Statistics List](../../../img/new_ui/dev/monitor/command-list.png)
- 待执行命令数：统计 t_ds_command 表的数据
- 执行失败的命令数：统计 t_ds_error_command 表的数据
- 待运行任务数：统计 Zookeeper 中 task_queue 的数据
- 待杀死任务数：统计 Zookeeper 中 task_kill 的数据
展示系统中的命令列表，数据来自`t_ds_command`表。
![Failure Command Statistics List](../../../img/new_ui/dev/monitor/failure-command-list.png)
展示系统中的失败命令列表，数据来自`t_ds_error_command`表。
### 审计日志
审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，他对于维护安全都很有用。
审计日志的记录提供了有关谁访问了系统，以及他或她在给定时间段内执行了哪些操作的信息，对于维护安全都很有用。
![audit-log](../../../img/new_ui/dev/monitor/audit-log.jpg)
![audit-log](../../../img/new_ui/dev/monitor/audit-log.png)

2
docs/docs/zh/guide/parameter/global.md
View File

@ -20,7 +20,7 @@
### 保存工作流，并设置全局参数
全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示：
全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示：
![global-parameter02](../../../../img/new_ui/dev/parameter/global_parameter02.png)

2
docs/docs/zh/guide/parameter/project-parameter.md
View File

@ -8,7 +8,7 @@
### 定义项目级别参数
在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示：
在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示：
![project-parameter01](../../../../img/new_ui/dev/parameter/project_parameter01.png)

2
docs/docs/zh/guide/parameter/startup-parameter.md
View File

@ -6,7 +6,7 @@
## 使用方式
启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。
启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。
## 任务样例

159
docs/docs/zh/guide/resource/configuration.md
View File

@ -26,77 +26,35 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
## 对接AWS S3
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下：
配置以下字段
```properties
......
resource.storage.type=S3
......
resource.aws.access.key.id=aws_access_key_id
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=aws_secret_access_key
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=us-west-2
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name=dolphinscheduler
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint=
......
```
## 对接分布式或远端对象存储
```yaml
aws:
s3:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
bucket.name: <bucket.name>
endpoint: <endpoint>
当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置：
```
### 配置 common.properties 文件
## 对接阿里云 OSS
在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
```properties
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
resource.storage.type=LOCAL
# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path=/tmp/dolphinscheduler
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=minioadmin
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=minioadmin
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=cn-north-1
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name=dolphinscheduler
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint=http://localhost:9000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id=<your-access-key-id>
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
@ -108,92 +66,27 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OSS
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
# alibaba cloud access key id, required if you set resource.storage.type=OBS
```
## 对接华为云 OBS
如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
```properties
# access key id, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.id=<your-access-key-id>
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
# access key secret, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
# oss bucket name, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
resource.hdfs.root.user=root
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler;
# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
resource.hdfs.fs.defaultFS=hdfs://localhost:8020
# whether to startup kerberos
hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path
java.security.krb5.conf.path=/opt/krb5.conf
# login user from keytab username
login.user.keytab.username=hdfs-mycluster@ESZ.COM
# login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab
# kerberos expire time, the unit is hour
kerberos.expire.time=2
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# resourcemanager port, the default value is 8088 if not specified
resource.manager.httpaddress.port=8088
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value;
# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s
# datasource encryption enable
datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
# Network IP gets priority, default inner outer
# Whether hive SQL is executed in the same session
support.hive.oneSession=false
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions;
# if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=true
# network interface preferred like eth0, default: empty
#dolphin.scheduler.network.interface.preferred=
# network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default
# system env path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
# development state
development.state=false
# rpc port
alert.rpc.port=50052
# way to collect applicationId: log(original regex match), aop
appId.collect: log
```
> **注意**：
>
> * 如果只配置了 `api-server/conf/common.properties` 的文件，则只是开启了资源上传的操作，并不能满足正常使用。如果想要在工作流中执行相关文件则需要额外配置 `worker-server/conf/common.properties`。
> * 如果用到资源上传的功能，那么[安装部署](../installation/standalone.md)中，部署用户需要有这部分的操作权限。
> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过次步骤。
> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话，需要开启 HDFS 类型的资源上传，同时需要将 Hadoop 集群下的 `core-site.xml` 和 `hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`，非 NameNode HA 跳过此步骤。

2
docs/docs/zh/guide/task/appendix.md
View File

@ -8,7 +8,7 @@
|----------|--------------------------------------------------------------------------------------------------------------------------------------|
| 任务名称 | 任务的名称，同一个工作流定义中的节点名称不能重复。 |
| 运行标志 | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。 |
| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。 |
| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。 |
| 描述 | 当前节点的功能描述。 |
| 任务优先级 | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。 |
| Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。 |

6
docs/docs/zh/guide/task/datafactory.md
View File

@ -19,10 +19,10 @@ DolphinScheduler DataFactory 组件的功能:
### 应用权限设置
首先打开当前`Subcription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。
![Subcription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
首先打开当前`Subscription`页面，点击`Access control (IAM)`，再点击`Add role assignment`进入授权页面。
![Subscription-IAM](../../../../img/tasks/demo/datafactory_auth1.png)
首先选择`Contributor`角色足够满足调用数据工厂。然后选择`Members`页面，再选择`Select members`，检索APP名称或APP的`Object ID`并添加，从给指定APP添加权限.
![Subcription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
![Subscription-Role](../../../../img/tasks/demo/datafactory_auth2.png)
## 环境配置

17
docs/docs/zh/guide/task/dms.md
View File

@ -73,14 +73,17 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS
## 环境配置
需要进行AWS的一些配置，修改`common.properties`中的以下配置信息
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
```yaml
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=<AWS REGION>
dms:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
endpoint: <endpoint>
```

2
docs/docs/zh/guide/task/mlflow.md
View File

@ -139,7 +139,7 @@ mlflow server -h 0.0.0.0 -p 5000 --serve-artifacts --backend-store-uri sqlite://
### 内置算法仓库配置
如果遇到github无法访问的情况，可以修改`commom.properties`配置文件的以下字段，将github地址替换能访问的地址。
如果遇到github无法访问的情况，可以修改`common.properties`配置文件的以下字段，将github地址替换能访问的地址。
```yaml
# mlflow task plugin preset repository

17
docs/docs/zh/guide/task/sagemaker.md
View File

@ -33,14 +33,17 @@ DolphinScheduler SageMaker 组件的功能:
## 环境配置
需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
```yaml
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region=<AWS REGION>
sagemaker:
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
credentials.provider.type: AWSStaticCredentialsProvider
access.key.id: <access.key.id>
access.key.secret: <access.key.secret>
region: <region>
endpoint: <endpoint>
```

BIN
docs/img/new_ui/dev/monitor/alert-server.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 135 KiB

BIN
docs/img/new_ui/dev/monitor/audit-log.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 222 KiB

BIN
docs/img/new_ui/dev/monitor/audit-log.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 114 KiB

BIN
docs/img/new_ui/dev/monitor/command-list.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 107 KiB

BIN
docs/img/new_ui/dev/monitor/failure-command-list.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 106 KiB

BIN
docs/img/new_ui/dev/monitor/master.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 89 KiB

After

Width:  |  Height:  |  Size: 134 KiB

BIN
docs/img/new_ui/dev/monitor/statistics.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 61 KiB

BIN
docs/img/new_ui/dev/monitor/worker.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 88 KiB

After

Width:  |  Height:  |  Size: 139 KiB

BIN
docs/img/new_ui/dev/parameter/context-subprocess02.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 137 KiB

After

Width:  |  Height:  |  Size: 327 KiB

BIN
docs/img/new_ui/dev/parameter/context_parameter04.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 166 KiB

After

Width:  |  Height:  |  Size: 338 KiB

BIN
docs/img/new_ui/dev/parameter/global_parameter02.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 112 KiB

After

Width:  |  Height:  |  Size: 326 KiB

BIN
docs/img/new_ui/dev/parameter/project_parameter01.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 3.1 MiB

After

Width:  |  Height:  |  Size: 118 KiB

BIN
docs/img/new_ui/dev/parameter/startup_parameter02.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 346 KiB

After

Width:  |  Height:  |  Size: 196 KiB

BIN
docs/img/new_ui/dev/parameter/startup_parameter04.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 347 KiB

After

Width:  |  Height:  |  Size: 195 KiB

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceAlertChannel.java
View File

@ -40,7 +40,7 @@ public final class VoiceAlertChannel implements AlertChannel {
Map<String, String> paramsMap = info.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "aliyun-voice params is null");
return new AlertResult(false, "aliyun-voice params is null");
}
VoiceParam voiceParam = buildVoiceParam(paramsMap);
return new VoiceSender(voiceParam).send();

4
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/main/java/org/apache/dolphinscheduler/plugin/alert/voice/VoiceSender.java
View File

@ -46,7 +46,7 @@ public final class VoiceSender {
public AlertResult send() {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
try {
Client client = createClient(voiceParam.getConnection());
SingleCallByTtsRequest singleCallByTtsRequest = new SingleCallByTtsRequest()
@ -61,7 +61,7 @@ public final class VoiceSender {
}
SingleCallByTtsResponseBody body = response.getBody();
if (body.code.equalsIgnoreCase("ok")) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage(body.getCallId());
} else {
alertResult.setMessage(body.getMessage());

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/java/VoiceSenderTest.java
View File

@ -46,7 +46,7 @@ class VoiceSenderTest {
VoiceSender weChatSender = new VoiceSender(voiceParam);
AlertResult alertResult = weChatSender.send();
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java
View File

@ -35,6 +35,6 @@ public interface AlertChannel {
AlertResult process(AlertInfo info);
default @NonNull AlertResult closeAlert(AlertInfo info) {
return new AlertResult("true", "no need to close alert");
return new AlertResult(true, "no need to close alert");
}
}

8
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertData.java
View File

@ -53,14 +53,6 @@ public class AlertData {
*/
private String log;
/**
* 0 do not send warning;
* 1 send if process success;
* 2 send if process failed;
* 3 send if process ends, whatever the result;
*/
private int warnType;
/**
* AlertType#code
*/

14
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertResult.java
View File

@ -33,15 +33,19 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class AlertResult {
/**
* todo: use enum
* false or true
*/
private String status;
private boolean success;
/**
* alert result message, each plugin can have its own message
*/
private String message;
public static AlertResult success() {
return new AlertResult(true, null);
}
public static AlertResult fail(String message) {
return new AlertResult(false, message);
}
}

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java
View File

@ -31,7 +31,7 @@ public final class DingTalkAlertChannel implements AlertChannel {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "ding talk params is null");
return new AlertResult(false, "ding talk params is null");
}
return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent());
}

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java
View File

@ -126,7 +126,7 @@ public final class DingTalkSender {
private AlertResult checkSendDingTalkSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
if (null == result) {
alertResult.setMessage("send ding talk msg error");
@ -140,7 +140,7 @@ public final class DingTalkSender {
return alertResult;
}
if (sendMsgResponse.errcode == 0) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage("send ding talk msg success");
return alertResult;
}
@ -164,7 +164,7 @@ public final class DingTalkSender {
} catch (Exception e) {
log.info("send ding talk alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage("send ding talk alert fail.");
}
return alertResult;

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java
View File

@ -52,7 +52,7 @@ public class DingTalkSenderTest {
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true");
dingTalkSender = new DingTalkSender(dingTalkConfig);
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test");
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertEquals(false, alertResult.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

10
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java
View File

@ -35,24 +35,20 @@ public final class EmailAlertChannel implements AlertChannel {
AlertData alert = info.getAlertData();
Map<String, String> paramsMap = info.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "mail params is null");
return new AlertResult(false, "mail params is null");
}
MailSender mailSender = new MailSender(paramsMap);
AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent());
boolean flag;
if (alertResult == null) {
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage("alert send error.");
log.info("alert send error : {}", alertResult.getMessage());
return alertResult;
}
flag = Boolean.parseBoolean(String.valueOf(alertResult.getStatus()));
if (flag) {
if (alertResult.isSuccess()) {
log.info("alert send success");
alertResult.setMessage("email send success.");
} else {

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java
View File

@ -154,7 +154,7 @@ public final class MailSender {
*/
public AlertResult sendMails(List<String> receivers, List<String> receiverCcs, String title, String content) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
// if there is no receivers && no receiversCc, no need to process
if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiverCcs)) {
@ -201,7 +201,7 @@ public final class MailSender {
attachment(title, content, partContent);
alertResult.setStatus("true");
alertResult.setSuccess(true);
return alertResult;
} catch (Exception e) {
handleException(alertResult, e);
@ -380,7 +380,7 @@ public final class MailSender {
email.setDebug(true);
email.send();
alertResult.setStatus("true");
alertResult.setSuccess(true);
return alertResult;
}

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java
View File

@ -66,7 +66,7 @@ public class EmailAlertChannelTest {
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult = emailAlertChannel.process(alertInfo);
Assertions.assertNotNull(alertResult);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
public String getEmailAlertParams() {

14
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java
View File

@ -77,7 +77,7 @@ public class MailUtilsTest {
AlertResult alertResult = mailSender.sendMails(
"Mysql Exception",
content);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
@ -107,7 +107,7 @@ public class MailUtilsTest {
emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd");
mailSender = new MailSender(emailConfig);
AlertResult alertResult = mailSender.sendMails(title, content);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
public String list2String() {
@ -142,24 +142,24 @@ public class MailUtilsTest {
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp());
mailSender = new MailSender(emailConfig);
AlertResult alertResult = mailSender.sendMails(title, content);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
public void testAttachmentFile() throws Exception {
public void testAttachmentFile() {
String content = list2String();
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp());
mailSender = new MailSender(emailConfig);
AlertResult alertResult = mailSender.sendMails("gaojing", content);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
public void testTableAttachmentFile() throws Exception {
public void testTableAttachmentFile() {
String content = list2String();
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp());
mailSender = new MailSender(emailConfig);
AlertResult alertResult = mailSender.sendMails("gaojing", content);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java
View File

@ -31,7 +31,7 @@ public final class FeiShuAlertChannel implements AlertChannel {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "fei shu params is null");
return new AlertResult(false, "fei shu params is null");
}
return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData);
}

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java
View File

@ -80,7 +80,7 @@ public final class FeiShuSender {
public static AlertResult checkSendFeiShuSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
if (org.apache.commons.lang3.StringUtils.isBlank(result)) {
alertResult.setMessage("send fei shu msg error");
@ -95,7 +95,7 @@ public final class FeiShuSender {
return alertResult;
}
if (sendMsgResponse.statusCode == 0) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage("send fei shu msg success");
return alertResult;
}
@ -136,7 +136,7 @@ public final class FeiShuSender {
} catch (Exception e) {
log.info("send fei shu alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage("send fei shu alert fail.");
}
return alertResult;

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java
View File

@ -43,7 +43,7 @@ public class FeiShuSenderTest {
alertData.setContent("feishu test content");
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData);
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
@ -87,12 +87,12 @@ public class FeiShuSenderTest {
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
AlertResult alertResult = feiShuSender.checkSendFeiShuSendMsgResult("");
Assertions.assertFalse(Boolean.valueOf(alertResult.getStatus()));
Assertions.assertFalse(alertResult.isSuccess());
AlertResult alertResult2 = feiShuSender.checkSendFeiShuSendMsgResult("123");
Assertions.assertEquals("send fei shu msg fail", alertResult2.getMessage());
String response = "{\"StatusCode\":\"0\",\"extra\":\"extra\",\"StatusMessage\":\"StatusMessage\"}";
AlertResult alertResult3 = feiShuSender.checkSendFeiShuSendMsgResult(response);
Assertions.assertTrue(Boolean.valueOf(alertResult3.getStatus()));
Assertions.assertTrue(alertResult3.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java
View File

@ -31,7 +31,7 @@ public final class HttpAlertChannel implements AlertChannel {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "http params is null");
return new AlertResult(false, "http params is null");
}
return new HttpSender(paramsMap).send(alertData.getContent());

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java
View File

@ -92,18 +92,18 @@ public final class HttpSender {
}
if (httpRequest == null) {
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage("Request types are not supported");
return alertResult;
}
try {
String resp = this.getResponseString(httpRequest);
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage(resp);
} catch (Exception e) {
log.error("send http alert msg exception : {}", e.getMessage());
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage(
String.format("Send http request alert failed: %s", e.getMessage()));
}

4
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java
View File

@ -62,9 +62,9 @@ public class HttpAlertChannelTest {
// HttpSender(paramsMap).send(alertData.getContent()); already test in HttpSenderTest.sendTest. so we can mock
// it
doReturn(new AlertResult("true", "success")).when(alertChannel).process(any());
doReturn(new AlertResult(true, "success")).when(alertChannel).process(any());
AlertResult alertResult = alertChannel.process(alertInfo);
Assertions.assertEquals("true", alertResult.getStatus());
Assertions.assertTrue(alertResult.isSuccess());
}
/**

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSenderTest.java
View File

@ -46,7 +46,7 @@ public class HttpSenderTest {
HttpSender httpSender = spy(new HttpSender(paramsMap));
doReturn("success").when(httpSender).getResponseString(any());
AlertResult alertResult = httpSender.send("Fault tolerance warning");
Assertions.assertEquals("true", alertResult.getStatus());
Assertions.assertTrue(alertResult.isSuccess());
Assertions.assertTrue(httpSender.getRequestUrl().contains(url));
Assertions.assertTrue(httpSender.getRequestUrl().contains(contentField));
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

4
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java
View File

@ -30,8 +30,8 @@ public final class PagerDutyAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> alertParams = alertInfo.getAlertParams();
if (alertParams == null || alertParams.size() == 0) {
return new AlertResult("false", "PagerDuty alert params is empty");
if (alertParams == null || alertParams.isEmpty()) {
return new AlertResult(false, "PagerDuty alert params is empty");
}
return new PagerDutySender(alertParams).sendPagerDutyAlter(alertData.getTitle(), alertData.getContent());

4
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java
View File

@ -53,7 +53,7 @@ public final class PagerDutySender {
public AlertResult sendPagerDutyAlter(String title, String content) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
alertResult.setMessage("send pager duty alert fail.");
try {
@ -83,7 +83,7 @@ public final class PagerDutySender {
String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8);
try {
if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage("send pager duty alert success");
} else {
alertResult.setMessage(

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySenderTest.java
View File

@ -39,6 +39,6 @@ public class PagerDutySenderTest {
public void testSend() {
PagerDutySender pagerDutySender = new PagerDutySender(pagerDutyConfig);
AlertResult alertResult = pagerDutySender.sendPagerDutyAlter("pagerduty test title", "pagerduty test content");
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertChannel.java
View File

@ -31,7 +31,7 @@ public final class PrometheusAlertChannel implements AlertChannel {
AlertData alertData = info.getAlertData();
Map<String, String> paramsMap = info.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "prometheus alert manager params is null");
return new AlertResult(false, "prometheus alert manager params is null");
}
return new PrometheusAlertSender(paramsMap).sendMessage(alertData);

12
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java
View File

@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.alert.api.HttpServiceRetryStrategy;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
@ -64,11 +65,10 @@ public class PrometheusAlertSender {
String resp = sendMsg(alertData);
return checkSendAlertManageMsgResult(resp);
} catch (Exception e) {
String errorMsg = String.format("send prometheus alert manager alert error, exception: %s", e.getMessage());
log.error(errorMsg);
log.error("Send prometheus alert manager alert error", e);
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage(errorMsg);
alertResult.setSuccess(false);
alertResult.setMessage(ExceptionUtils.getMessage(e));
}
return alertResult;
}
@ -106,10 +106,10 @@ public class PrometheusAlertSender {
public AlertResult checkSendAlertManageMsgResult(String resp) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
if (Objects.equals(resp, PrometheusAlertConstants.ALERT_SUCCESS)) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage("prometheus alert manager send success");
return alertResult;
}

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSenderTest.java
View File

@ -55,17 +55,17 @@ public class PrometheusAlertSenderTest {
" }]");
PrometheusAlertSender sender = new PrometheusAlertSender(config);
AlertResult result = sender.sendMessage(alertData);
Assertions.assertEquals("false", result.getStatus());
Assertions.assertFalse(result.isSuccess());
}
@Test
public void testCheckSendAlertManageMsgResult() {
PrometheusAlertSender prometheusAlertSender = new PrometheusAlertSender(config);
AlertResult alertResult1 = prometheusAlertSender.checkSendAlertManageMsgResult("");
Assertions.assertFalse(Boolean.parseBoolean(alertResult1.getStatus()));
Assertions.assertFalse(alertResult1.isSuccess());
Assertions.assertEquals("prometheus alert manager send fail, resp is ", alertResult1.getMessage());
AlertResult alertResult2 = prometheusAlertSender.checkSendAlertManageMsgResult("alert success");
Assertions.assertTrue(Boolean.parseBoolean(alertResult2.getStatus()));
Assertions.assertTrue(alertResult2.isSuccess());
Assertions.assertEquals("prometheus alert manager send success", alertResult2.getMessage());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java
View File

@ -33,7 +33,7 @@ public final class ScriptAlertChannel implements AlertChannel {
AlertData alertData = alertinfo.getAlertData();
Map<String, String> paramsMap = alertinfo.getAlertParams();
if (MapUtils.isEmpty(paramsMap)) {
return new AlertResult("false", "script params is empty");
return new AlertResult(false, "script params is empty");
}
return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle(), alertData.getContent());
}

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java
View File

@ -56,7 +56,7 @@ public final class ScriptSender {
}
// If it is another type of alarm script can be added here, such as python
alertResult.setStatus("false");
alertResult.setSuccess(false);
log.error("script type error: {}", scriptType);
alertResult.setMessage("script type error : " + scriptType);
return alertResult;
@ -64,7 +64,7 @@ public final class ScriptSender {
private AlertResult executeShellScript(String title, String content) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setSuccess(false);
if (Boolean.TRUE.equals(OSUtils.isWindows())) {
alertResult.setMessage("shell script not support windows os");
return alertResult;
@ -111,7 +111,7 @@ public final class ScriptSender {
int exitCode = ProcessUtils.executeScript(cmd);
if (exitCode == 0) {
alertResult.setStatus("true");
alertResult.setSuccess(true);
alertResult.setMessage("send script alert msg success");
return alertResult;
}

16
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSenderTest.java
View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.plugin.alert.script;
import static org.junit.jupiter.api.Assertions.assertFalse;
import org.apache.dolphinscheduler.alert.api.AlertResult;
import java.util.HashMap;
@ -48,9 +50,9 @@ public class ScriptSenderTest {
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult;
alertResult = scriptSender.sendScriptAlert("test title Kris", "test content");
Assertions.assertEquals("true", alertResult.getStatus());
Assertions.assertTrue(alertResult.isSuccess());
alertResult = scriptSender.sendScriptAlert("error msg title", "test content");
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
@ -58,7 +60,7 @@ public class ScriptSenderTest {
scriptConfig.put(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, "' ; calc.exe ; '");
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult = scriptSender.sendScriptAlert("test title Kris", "test content");
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
@ -67,7 +69,7 @@ public class ScriptSenderTest {
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult;
alertResult = scriptSender.sendScriptAlert("test user params NPE", "test content");
Assertions.assertEquals("true", alertResult.getStatus());
Assertions.assertTrue(alertResult.isSuccess());
}
@Test
@ -76,7 +78,7 @@ public class ScriptSenderTest {
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult;
alertResult = scriptSender.sendScriptAlert("test path NPE", "test content");
Assertions.assertEquals("false", alertResult.getStatus());
Assertions.assertFalse(alertResult.isSuccess());
}
@Test
@ -85,7 +87,7 @@ public class ScriptSenderTest {
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult;
alertResult = scriptSender.sendScriptAlert("test path NPE", "test content");
Assertions.assertEquals("false", alertResult.getStatus());
assertFalse(alertResult.isSuccess());
Assertions.assertTrue(alertResult.getMessage().contains("shell script is invalid, only support .sh file"));
}
@ -95,7 +97,7 @@ public class ScriptSenderTest {
ScriptSender scriptSender = new ScriptSender(scriptConfig);
AlertResult alertResult;
alertResult = scriptSender.sendScriptAlert("test type is error", "test content");
Assertions.assertEquals("false", alertResult.getStatus());
assertFalse(alertResult.isSuccess());
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

6
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java
View File

@ -30,11 +30,11 @@ public final class SlackAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> alertParams = alertInfo.getAlertParams();
if (alertParams == null || alertParams.size() == 0) {
return new AlertResult("false", "Slack alert params is empty");
if (alertParams == null || alertParams.isEmpty()) {
return new AlertResult(false, "Slack alert params is empty");
}
SlackSender slackSender = new SlackSender(alertParams);
String response = slackSender.sendMessage(alertData.getTitle(), alertData.getContent());
return new AlertResult("ok".equals(response) ? "true" : "false", response);
return new AlertResult("ok".equals(response), response);
}
}

21
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java
View File

@ -30,7 +30,7 @@ public final class TelegramAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo info) {
Map<String, String> alertParams = info.getAlertParams();
if (alertParams == null || alertParams.isEmpty()) {
return new AlertResult("false", "Telegram alert params is empty");
return AlertResult.fail("Telegram alert params is empty");
}
AlertData data = info.getAlertData();
return new TelegramSender(alertParams).sendMessage(data);

Some files were not shown because too many files have changed in this diff Show More