Merge branch 'dev' into issues/15591
This commit is contained in:
JohnHuang
GitHub
commit
6c94eb36bf
|
|
@ -45,7 +45,8 @@ github:
|
|||
- Unit Test
|
||||
- E2E
|
||||
- Docs
|
||||
- "Mergeable: milestone-label-check"
|
||||
- Frontend Build
|
||||
# - "Mergeable: milestone-label-check"
|
||||
required_pull_request_reviews:
|
||||
dismiss_stale_reviews: true
|
||||
required_approving_review_count: 1
|
||||
required_approving_review_count: 2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
name: DSIP
|
||||
description: Suggest an idea for this project
|
||||
title: "[DSIP-][Module Name] DSIP title"
|
||||
labels: [ "DSIP", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first
|
||||
to see whether the same DSIP was created already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no
|
||||
similar DSIP.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Motivation
|
||||
description: Why you want to do this change?
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Design Detail
|
||||
description: Your design.
|
||||
placeholder: >
|
||||
It's better to provide a detailed design, such as the design of the interface, the design of the database, etc.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Compatibility, Deprecation, and Migration Plan
|
||||
description: >
|
||||
If this feature is related to compatibility, deprecation, or migration, please describe it here.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Test Plan
|
||||
description: >
|
||||
How to test this improvement.
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
---
|
||||
version: 2
|
||||
mergeable:
|
||||
# we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
|
||||
# otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
|
||||
# GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
|
||||
# which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
|
||||
- when: pull_request.opened, pull_request.reopened, pull_request.synchronize
|
||||
name: sync-sql-ddl
|
||||
validate:
|
||||
# Sql files must change synchronize
|
||||
- do: dependent
|
||||
files:
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
|
||||
message: 'Sql files not change synchronize'
|
||||
# Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
|
||||
fail:
|
||||
- do: comment
|
||||
payload:
|
||||
body: >
|
||||
:warning: This PR do not change database DDL synchronize.
|
||||
leave_old_comment: false
|
||||
- do: labels
|
||||
add: 'sql not sync'
|
||||
# Remove labels 'sql not sync' if pass
|
||||
pass:
|
||||
- do: labels
|
||||
delete: 'sql not sync'
|
||||
|
||||
- when: pull_request.*
|
||||
name: milestone-label-check
|
||||
validate:
|
||||
- do: milestone
|
||||
no_empty:
|
||||
enabled: false # Cannot be empty when true.
|
||||
message: 'Milestone is required and cannot be empty.'
|
||||
- do: label
|
||||
and:
|
||||
- must_include:
|
||||
regex: 'feature|bug|improvement|document|chore|revert'
|
||||
message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
|
||||
- must_include:
|
||||
regex: 'ready-to-merge'
|
||||
message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'
|
||||
|
|
@ -106,10 +106,14 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: cluster-test-mysql
|
||||
script: .github/workflows/cluster-test/mysql/start-job.sh
|
||||
- name: cluster-test-postgresql
|
||||
script: .github/workflows/cluster-test/postgresql/start-job.sh
|
||||
- name: cluster-test-mysql-with-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-mysql-with-mysql-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-with-postgresql-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
|
|||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
|
|
@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME
|
|||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Create database
|
||||
mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
34
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
vendored
Normal file
34
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mysql:
|
||||
container_name: mysql
|
||||
image: mysql:5.7.36
|
||||
command: --default-authentication-plugin=mysql_native_password
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: 123456
|
||||
ports:
|
||||
- "3306:3306"
|
||||
healthcheck:
|
||||
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
57
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
vendored
Executable file
57
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
vendored
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-mysql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export SPRING_DATASOURCE_USERNAME=root
|
||||
export SPRING_DATASOURCE_PASSWORD=123456
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=root
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=123456
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
|
@ -18,16 +18,16 @@
|
|||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile .
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
|
||||
RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
29
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
29
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_mysql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
108
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
vendored
Normal file
108
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
vendored
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
39
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
vendored
Normal file
39
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
41
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
vendored
Normal file
41
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
35
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
vendored
Normal file
35
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
container_name: postgres
|
||||
image: postgres:14.1
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dolphinscheduler
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
||||
57
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
vendored
Normal file
57
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
vendored
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export SPRING_DATASOURCE_USERNAME=postgres
|
||||
export SPRING_DATASOURCE_PASSWORD=postgres
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=jdbc
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=postgres
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
58
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
vendored
Normal file
58
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
33
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
vendored
Normal file
33
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
|
|||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
29
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
29
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_postgresql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
109
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
vendored
Normal file
109
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
vendored
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -18,16 +18,16 @@
|
|||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile .
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -25,9 +25,6 @@ on:
|
|||
- '.github/workflows/frontend.yml'
|
||||
- 'dolphinscheduler-ui/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/frontend.yml'
|
||||
- 'dolphinscheduler-ui/**'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
|
@ -38,10 +35,25 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Frontend-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ ubuntu-latest, macos-latest ]
|
||||
|
|
@ -56,9 +68,28 @@ jobs:
|
|||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 16
|
||||
- name: Compile and Build
|
||||
- name: Compile and Build on ${{ matrix.os }}
|
||||
run: |
|
||||
set -x
|
||||
npm install pnpm@7 -g
|
||||
pnpm install
|
||||
pnpm run lint
|
||||
pnpm run build:prod
|
||||
result:
|
||||
name: Frontend Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ build, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Frontend Build!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.build.result }} != 'success' ]]; then
|
||||
echo "Frontend Build Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
---
|
||||
#version: 2
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
name: "Mergeable"
|
||||
|
||||
jobs:
|
||||
result:
|
||||
name: "Mergeable: milestone-label-check"
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
echo "Temporary skipping this check"
|
||||
|
||||
#mergeable:
|
||||
# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
|
||||
# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
|
||||
# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
|
||||
# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
|
||||
# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
|
||||
# name: sync-sql-ddl
|
||||
# validate:
|
||||
# # Sql files must change synchronize
|
||||
# - do: dependent
|
||||
# files:
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
|
||||
# message: 'Sql files not change synchronize'
|
||||
# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
|
||||
# fail:
|
||||
# - do: comment
|
||||
# payload:
|
||||
# body: >
|
||||
# :warning: This PR do not change database DDL synchronize.
|
||||
# leave_old_comment: false
|
||||
# - do: labels
|
||||
# add: 'sql not sync'
|
||||
# # Remove labels 'sql not sync' if pass
|
||||
# pass:
|
||||
# - do: labels
|
||||
# delete: 'sql not sync'
|
||||
#
|
||||
# - when: pull_request.*
|
||||
# name: milestone-label-check
|
||||
# validate:
|
||||
# - do: milestone
|
||||
# no_empty:
|
||||
# enabled: false # Cannot be empty when true.
|
||||
# message: 'Milestone is required and cannot be empty.'
|
||||
# - do: label
|
||||
# and:
|
||||
# - must_include:
|
||||
# regex: 'feature|bug|improvement|document|chore|revert'
|
||||
# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
|
||||
# - must_include:
|
||||
# regex: 'ready-to-merge'
|
||||
# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'
|
||||
|
|
@ -174,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| image.master | string | `"dolphinscheduler-master"` | master image |
|
||||
| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent |
|
||||
| image.pullSecret | string | `""` | Specify a imagePullSecrets |
|
||||
| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler |
|
||||
| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler |
|
||||
| image.tag | string | `"latest"` | Docker image version for the DolphinScheduler |
|
||||
| image.tools | string | `"dolphinscheduler-tools"` | tools image |
|
||||
| image.worker | string | `"dolphinscheduler-worker"` | worker image |
|
||||
|
|
@ -200,9 +200,9 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| master.env.MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER | string | `"true"` | Master kill application when handle failover |
|
||||
| master.env.MASTER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Master max heartbeat interval |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open master overload protection |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
|
||||
| master.env.MASTER_STATE_WHEEL_INTERVAL | string | `"5s"` | master state wheel interval, the unit is second |
|
||||
| master.env.MASTER_TASK_COMMIT_INTERVAL | string | `"1s"` | master commit task interval, the unit is second |
|
||||
|
|
@ -301,9 +301,9 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| worker.env.WORKER_HOST_WEIGHT | string | `"100"` | Worker host weight to dispatch tasks |
|
||||
| worker.env.WORKER_MAX_HEARTBEAT_INTERVAL | string | `"10s"` | Worker heartbeat interval |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_ENABLED | bool | `false` | If set true, will open worker overload protection |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
|
||||
| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. |
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ data:
|
|||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
common_properties: |-
|
||||
common.properties: |-
|
||||
{{- if index .Values.conf "common" }}
|
||||
{{- range $key, $value := index .Values.conf "common" }}
|
||||
{{- if and $.Values.minio.enabled }}
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ spec:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
- name: config-volume
|
||||
mountPath: /opt/dolphinscheduler/conf/common.properties
|
||||
subPath: common_properties
|
||||
subPath: common.properties
|
||||
volumes:
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
{{- if .Values.alert.persistentVolumeClaim.enabled }}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ spec:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
- name: config-volume
|
||||
mountPath: /opt/dolphinscheduler/conf/common.properties
|
||||
subPath: common_properties
|
||||
subPath: common.properties
|
||||
{{- if .Values.api.taskTypeFilter.enabled }}
|
||||
- name: config-volume
|
||||
mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ spec:
|
|||
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
|
||||
- name: config-volume
|
||||
mountPath: /opt/dolphinscheduler/conf/common.properties
|
||||
subPath: common_properties
|
||||
subPath: common.properties
|
||||
{{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
|
||||
volumes:
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ spec:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
|
||||
- name: config-volume
|
||||
mountPath: /opt/dolphinscheduler/conf/common.properties
|
||||
subPath: common_properties
|
||||
subPath: common.properties
|
||||
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ initImage:
|
|||
|
||||
image:
|
||||
# -- Docker image repository for the DolphinScheduler
|
||||
registry: apache/dolphinscheduler
|
||||
registry: apache
|
||||
# -- Docker image version for the DolphinScheduler
|
||||
tag: latest
|
||||
# -- Image pull policy. Options: Always, Never, IfNotPresent
|
||||
|
|
@ -508,10 +508,10 @@ master:
|
|||
MASTER_STATE_WHEEL_INTERVAL: "5s"
|
||||
# -- If set true, will open master overload protection
|
||||
MASTER_SERVER_LOAD_PROTECTION_ENABLED: false
|
||||
# -- Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow.
|
||||
MASTER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow.
|
||||
MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
|
||||
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
|
||||
MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
|
||||
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
|
||||
|
|
@ -629,10 +629,10 @@ worker:
|
|||
env:
|
||||
# -- If set true, will open worker overload protection
|
||||
WORKER_SERVER_LOAD_PROTECTION_ENABLED: false
|
||||
# -- Worker max cpu usage, when the worker's cpu usage is smaller then this value, worker server can be dispatched tasks.
|
||||
WORKER_SERVER_LOAD_PROTECTION_MAX_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Worker max jvm memory usage , when the worker's jvm memory usage is smaller then this value, worker server can be dispatched tasks.
|
||||
WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
|
||||
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
|
||||
WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks.
|
||||
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.7
|
||||
# -- Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
|
||||
|
|
|
|||
|
|
@ -110,7 +110,8 @@ The directory structure of DolphinScheduler is as follows:
|
|||
|
||||
dolphinscheduler-daemon.sh is responsible for DolphinScheduler startup and shutdown.
|
||||
Essentially, start-all.sh or stop-all.sh startup and shutdown the cluster via dolphinscheduler-daemon.sh.
|
||||
Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical situation of resources.
|
||||
Currently, DolphinScheduler just makes a basic config, remember to config further JVM options based on your practical
|
||||
situation of resources.
|
||||
|
||||
Default simplified parameters are:
|
||||
|
||||
|
|
@ -128,44 +129,47 @@ export DOLPHINSCHEDULER_OPTS="
|
|||
"
|
||||
```
|
||||
|
||||
> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to communicate).
|
||||
> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6.
|
||||
> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DolphinScheduler dependent on Netty to
|
||||
> communicate).
|
||||
> If add "-Djava.net.preferIPv6Addresses=true" will use ipv6 address, if add "-Djava.net.preferIPv4Addresses=true" will
|
||||
> use ipv4 address, if doesn't set the two parameter will use ipv4 or ipv6.
|
||||
|
||||
### Database connection related configuration
|
||||
|
||||
DolphinScheduler uses Spring Hikari to manage database connections, configuration file location:
|
||||
|
||||
|Service| Configuration file |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/application.yaml`|
|
||||
|Api Server| `api-server/conf/application.yaml`|
|
||||
|Worker Server| `worker-server/conf/application.yaml`|
|
||||
|Alert Server| `alert-server/conf/application.yaml`|
|
||||
| Service | Configuration file |
|
||||
|---------------|---------------------------------------|
|
||||
| Master Server | `master-server/conf/application.yaml` |
|
||||
| Api Server | `api-server/conf/application.yaml` |
|
||||
| Worker Server | `worker-server/conf/application.yaml` |
|
||||
| Alert Server | `alert-server/conf/application.yaml` |
|
||||
|
||||
The default configuration is as follows:
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
|spring.datasource.driver-class-name| org.postgresql.Driver |datasource driver|
|
||||
|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |datasource connection url|
|
||||
|spring.datasource.username|root|datasource username|
|
||||
|spring.datasource.password|root|datasource password|
|
||||
|spring.datasource.hikari.connection-test-query|select 1|validate connection by running the SQL|
|
||||
|spring.datasource.hikari.minimum-idle| 5| minimum connection pool size number|
|
||||
|spring.datasource.hikari.auto-commit|true|whether auto commit|
|
||||
|spring.datasource.hikari.pool-name|DolphinScheduler|name of the connection pool|
|
||||
|spring.datasource.hikari.maximum-pool-size|50| maximum connection pool size number|
|
||||
|spring.datasource.hikari.connection-timeout|30000|connection timeout|
|
||||
|spring.datasource.hikari.idle-timeout|600000|Maximum idle connection survival time|
|
||||
|spring.datasource.hikari.leak-detection-threshold|0|Connection leak detection threshold|
|
||||
|spring.datasource.hikari.initialization-fail-timeout|1|Connection pool initialization failed timeout|
|
||||
| Parameters | Default value | Description |
|
||||
|------------------------------------------------------|---------------------------------------------------|-----------------------------------------------|
|
||||
| spring.datasource.driver-class-name | org.postgresql.Driver | datasource driver |
|
||||
| spring.datasource.url | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | datasource connection url |
|
||||
| spring.datasource.username | root | datasource username |
|
||||
| spring.datasource.password | root | datasource password |
|
||||
| spring.datasource.hikari.connection-test-query | select 1 | validate connection by running the SQL |
|
||||
| spring.datasource.hikari.minimum-idle | 5 | minimum connection pool size number |
|
||||
| spring.datasource.hikari.auto-commit | true | whether auto commit |
|
||||
| spring.datasource.hikari.pool-name | DolphinScheduler | name of the connection pool |
|
||||
| spring.datasource.hikari.maximum-pool-size | 50 | maximum connection pool size number |
|
||||
| spring.datasource.hikari.connection-timeout | 30000 | connection timeout |
|
||||
| spring.datasource.hikari.idle-timeout | 600000 | Maximum idle connection survival time |
|
||||
| spring.datasource.hikari.leak-detection-threshold | 0 | Connection leak detection threshold |
|
||||
| spring.datasource.hikari.initialization-fail-timeout | 1 | Connection pool initialization failed timeout |
|
||||
|
||||
Note that DolphinScheduler also supports database configuration through `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
### Registry Related configuration
|
||||
|
||||
DolphinScheduler uses Zookeeper by default for cluster management, fault tolerance, event monitoring and other functions. Configuration file location:
|
||||
|Service| Configuration file |
|
||||
DolphinScheduler uses Zookeeper for cluster management, fault tolerance, event monitoring and other functions.
|
||||
Configuration file location:
|
||||
|Service| Configuration file |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/application.yaml`|
|
||||
|Api Server| `api-server/conf/application.yaml`|
|
||||
|
|
@ -173,17 +177,17 @@ DolphinScheduler uses Zookeeper by default for cluster management, fault toleran
|
|||
|
||||
The default configuration is as follows:
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
|registry.zookeeper.namespace|dolphinscheduler|namespace of zookeeper|
|
||||
|registry.zookeeper.connect-string|localhost:2181| the connection string of zookeeper|
|
||||
|registry.zookeeper.retry-policy.base-sleep-time|60ms|time to wait between subsequent retries|
|
||||
|registry.zookeeper.retry-policy.max-sleep|300ms|maximum time to wait between subsequent retries|
|
||||
|registry.zookeeper.retry-policy.max-retries|5|maximum retry times|
|
||||
|registry.zookeeper.session-timeout|30s|session timeout|
|
||||
|registry.zookeeper.connection-timeout|30s|connection timeout|
|
||||
|registry.zookeeper.block-until-connected|600ms|waiting time to block until the connection succeeds|
|
||||
|registry.zookeeper.digest|{username}:{password}|digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) |
|
||||
| Parameters | Default value | Description |
|
||||
|-------------------------------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| registry.zookeeper.namespace | dolphinscheduler | namespace of zookeeper |
|
||||
| registry.zookeeper.connect-string | localhost:2181 | the connection string of zookeeper |
|
||||
| registry.zookeeper.retry-policy.base-sleep-time | 60ms | time to wait between subsequent retries |
|
||||
| registry.zookeeper.retry-policy.max-sleep | 300ms | maximum time to wait between subsequent retries |
|
||||
| registry.zookeeper.retry-policy.max-retries | 5 | maximum retry times |
|
||||
| registry.zookeeper.session-timeout | 30s | session timeout |
|
||||
| registry.zookeeper.connection-timeout | 30s | connection timeout |
|
||||
| registry.zookeeper.block-until-connected | 600ms | waiting time to block until the connection succeeds |
|
||||
| registry.zookeeper.digest | {username}:{password} | digest of zookeeper to access znode, works only when acl is enabled, for more details please check [https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper doc) |
|
||||
|
||||
Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
|
|
@ -194,12 +198,12 @@ For JDBC Registry, please see more details on [link](https://github.com/apache/d
|
|||
|
||||
Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
|
||||
|
||||
|Service| Configuration file |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/common.properties`|
|
||||
|Api Server| `api-server/conf/common.properties`|
|
||||
|Worker Server| `worker-server/conf/common.properties`|
|
||||
|Alert Server| `alert-server/conf/common.properties`|
|
||||
| Service | Configuration file |
|
||||
|---------------|----------------------------------------|
|
||||
| Master Server | `master-server/conf/common.properties` |
|
||||
| Api Server | `api-server/conf/common.properties` |
|
||||
| Worker Server | `worker-server/conf/common.properties` |
|
||||
| Alert Server | `alert-server/conf/common.properties` |
|
||||
|
||||
The default configuration is as follows:
|
||||
|
||||
|
|
@ -240,43 +244,43 @@ The default configuration is as follows:
|
|||
|
||||
Location: `api-server/conf/application.yaml`
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
|server.port|12345|api service communication port|
|
||||
|server.servlet.session.timeout|120m|session timeout|
|
||||
|server.servlet.context-path|/dolphinscheduler/ |request path|
|
||||
|spring.servlet.multipart.max-file-size|1024MB|maximum file size|
|
||||
|spring.servlet.multipart.max-request-size|1024MB|maximum request size|
|
||||
|server.jetty.max-http-post-size|5000000|jetty maximum post size|
|
||||
|spring.banner.charset|UTF-8|message encoding|
|
||||
|spring.jackson.time-zone|UTC|time zone|
|
||||
|spring.jackson.date-format|"yyyy-MM-dd HH:mm:ss"|time format|
|
||||
|spring.messages.basename|i18n/messages|i18n config|
|
||||
|security.authentication.type|PASSWORD|authentication type|
|
||||
|security.authentication.ldap.user.admin|read-only-admin|admin user account when you log-in with LDAP|
|
||||
|security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls|
|
||||
|security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn|
|
||||
|security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username|
|
||||
|security.authentication.ldap.password|password|LDAP password|
|
||||
|security.authentication.ldap.user.identity-attribute|uid|LDAP user identity attribute|
|
||||
|security.authentication.ldap.user.email-attribute|mail|LDAP user email attribute|
|
||||
|security.authentication.ldap.user.not-exist-action|CREATE|action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY)|
|
||||
|security.authentication.ldap.ssl.enable|false|LDAP ssl switch|
|
||||
|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks file absolute path|
|
||||
|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks password|
|
||||
|security.authentication.casdoor.user.admin||admin user account when you log-in with Casdoor|
|
||||
|casdoor.endpoint||Casdoor server url|
|
||||
|casdoor.client-id||id in Casdoor|
|
||||
|casdoor.client-secret||secret in Casdoor|
|
||||
|casdoor.certificate||certificate in Casdoor|
|
||||
|casdoor.organization-name||organization name in Casdoor|
|
||||
|casdoor.application-name||application name in Casdoor|
|
||||
|casdoor.redirect-url||doplhinscheduler login url|
|
||||
|api.traffic.control.global.switch|false|traffic control global switch|
|
||||
|api.traffic.control.max-global-qps-rate|300|global max request number per second|
|
||||
|api.traffic.control.tenant-switch|false|traffic control tenant switch|
|
||||
|api.traffic.control.default-tenant-qps-rate|10|default tenant max request number per second|
|
||||
|api.traffic.control.customize-tenant-qps-rate||customize tenant max request number per second|
|
||||
| Parameters | Default value | Description |
|
||||
|-------------------------------------------------------|--------------------------------------|------------------------------------------------------------------------------------------------|
|
||||
| server.port | 12345 | api service communication port |
|
||||
| server.servlet.session.timeout | 120m | session timeout |
|
||||
| server.servlet.context-path | /dolphinscheduler/ | request path |
|
||||
| spring.servlet.multipart.max-file-size | 1024MB | maximum file size |
|
||||
| spring.servlet.multipart.max-request-size | 1024MB | maximum request size |
|
||||
| server.jetty.max-http-post-size | 5000000 | jetty maximum post size |
|
||||
| spring.banner.charset | UTF-8 | message encoding |
|
||||
| spring.jackson.time-zone | UTC | time zone |
|
||||
| spring.jackson.date-format | "yyyy-MM-dd HH:mm:ss" | time format |
|
||||
| spring.messages.basename | i18n/messages | i18n config |
|
||||
| security.authentication.type | PASSWORD | authentication type |
|
||||
| security.authentication.ldap.user.admin | read-only-admin | admin user account when you log-in with LDAP |
|
||||
| security.authentication.ldap.urls | ldap://ldap.forumsys.com:389/ | LDAP urls |
|
||||
| security.authentication.ldap.base.dn | dc=example,dc=com | LDAP base dn |
|
||||
| security.authentication.ldap.username | cn=read-only-admin,dc=example,dc=com | LDAP username |
|
||||
| security.authentication.ldap.password | password | LDAP password |
|
||||
| security.authentication.ldap.user.identity-attribute | uid | LDAP user identity attribute |
|
||||
| security.authentication.ldap.user.email-attribute | mail | LDAP user email attribute |
|
||||
| security.authentication.ldap.user.not-exist-action | CREATE | action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY) |
|
||||
| security.authentication.ldap.ssl.enable | false | LDAP ssl switch |
|
||||
| security.authentication.ldap.ssl.trust-store | ldapkeystore.jks | LDAP jks file absolute path |
|
||||
| security.authentication.ldap.ssl.trust-store-password | password | LDAP jks password |
|
||||
| security.authentication.casdoor.user.admin | | admin user account when you log-in with Casdoor |
|
||||
| casdoor.endpoint | | Casdoor server url |
|
||||
| casdoor.client-id | | id in Casdoor |
|
||||
| casdoor.client-secret | | secret in Casdoor |
|
||||
| casdoor.certificate | | certificate in Casdoor |
|
||||
| casdoor.organization-name | | organization name in Casdoor |
|
||||
| casdoor.application-name | | application name in Casdoor |
|
||||
| casdoor.redirect-url | | doplhinscheduler login url |
|
||||
| api.traffic.control.global.switch | false | traffic control global switch |
|
||||
| api.traffic.control.max-global-qps-rate | 300 | global max request number per second |
|
||||
| api.traffic.control.tenant-switch | false | traffic control tenant switch |
|
||||
| api.traffic.control.default-tenant-qps-rate | 10 | default tenant max request number per second |
|
||||
| api.traffic.control.customize-tenant-qps-rate | | customize tenant max request number per second |
|
||||
|
||||
### Master Server related configuration
|
||||
|
||||
|
|
@ -285,7 +289,6 @@ Location: `master-server/conf/application.yaml`
|
|||
| Parameters | Default value | Description |
|
||||
|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master listen port |
|
||||
| master.fetch-command-num | 10 | the number of commands fetched by master |
|
||||
| master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel |
|
||||
| master.exec-threads | 100 | master execute thread number to limit process instances in parallel |
|
||||
| master.dispatch-task-number | 3 | master dispatch task number per batch |
|
||||
|
|
@ -295,37 +298,40 @@ Location: `master-server/conf/application.yaml`
|
|||
| master.task-commit-interval | 1000 | master commit task interval, the unit is millisecond |
|
||||
| master.state-wheel-interval | 5 | time to check status |
|
||||
| master.server-load-protection.enabled | true | If set true, will open master overload protection |
|
||||
| master.server-load-protection.max-cpu-usage-percentage-thresholds | 0.7 | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-jvm-memory-usage-percentage-thresholds | 0.7 | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Master max JVM cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max system memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
|
||||
| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
|
||||
| master.failover-interval | 10 | failover interval, the unit is minute |
|
||||
| master.kill-application-when-task-failover | true | whether to kill yarn/k8s application when failover taskInstance |
|
||||
| master.registry-disconnect-strategy.strategy | stop | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely |
|
||||
| master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory |
|
||||
| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` |
|
||||
| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db |
|
||||
| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master |
|
||||
|
||||
### Worker Server related configuration
|
||||
|
||||
Location: `worker-server/conf/application.yaml`
|
||||
|
||||
| Parameters | Default value | Description |
|
||||
|--------------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| worker.listen-port | 1234 | worker-service listen port |
|
||||
| worker.exec-threads | 100 | worker-service execute thread number, used to limit the number of task instances in parallel |
|
||||
| worker.max-heartbeat-interval | 10s | worker-service max heartbeat interval |
|
||||
| worker.host-weight | 100 | worker host weight to dispatch tasks |
|
||||
| worker.server-load-protection.enabled | true | If set true will open worker overload protection |
|
||||
| worker.max-cpu-usage-percentage-thresholds.max-cpu-usage-percentage-thresholds | 0.7 | Master max cpu usage, when the master's cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-jvm-memory-usage-percentage-thresholds | 0.7 | Master max JVM memory usage , when the master's jvm memory usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.registry-disconnect-strategy.strategy | stop | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting |
|
||||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
|
||||
| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants |
|
||||
| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. |
|
||||
| Parameters | Default value | Description |
|
||||
|-----------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| worker.listen-port | 1234 | worker-service listen port |
|
||||
| worker.exec-threads | 100 | worker-service execute thread number, used to limit the number of task instances in parallel |
|
||||
| worker.max-heartbeat-interval | 10s | worker-service max heartbeat interval |
|
||||
| worker.host-weight | 100 | worker host weight to dispatch tasks |
|
||||
| worker.server-load-protection.enabled | true | If set true will open worker overload protection |
|
||||
| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Worker max JVM cpu usage, when the worker's jvm cpu usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Worker max system memory usage , when the worker's system memory usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Worker max disk usage , when the worker's disk usage is smaller then this value, master server can execute workflow. |
|
||||
| worker.registry-disconnect-strategy.strategy | stop | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting |
|
||||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
|
||||
| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants |
|
||||
| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. |
|
||||
|
||||
### Alert Server related configuration
|
||||
|
||||
|
|
@ -340,10 +346,10 @@ Location: `alert-server/conf/application.yaml`
|
|||
|
||||
This part describes quartz configs and configure them based on your practical situation and resources.
|
||||
|
||||
|Service| Configuration file |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/application.yaml`|
|
||||
|Api Server| `api-server/conf/application.yaml`|
|
||||
| Service | Configuration file |
|
||||
|---------------|---------------------------------------|
|
||||
| Master Server | `master-server/conf/application.yaml` |
|
||||
| Api Server | `api-server/conf/application.yaml` |
|
||||
|
||||
The default configuration is as follows:
|
||||
|
||||
|
|
@ -361,7 +367,8 @@ The default configuration is as follows:
|
|||
| spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate |
|
||||
| spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000 |
|
||||
|
||||
The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool configuration is different.
|
||||
The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool
|
||||
configuration is different.
|
||||
|
||||
The default quartz threadpool configuration in *Master Server* is as follows:
|
||||
|
||||
|
|
@ -372,7 +379,8 @@ The default quartz threadpool configuration in *Master Server* is as follows:
|
|||
| spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 |
|
||||
| spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool |
|
||||
|
||||
Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured as `QuartzZeroSizeThreadPool` which has zero thread;
|
||||
Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured
|
||||
as `QuartzZeroSizeThreadPool` which has zero thread;
|
||||
The default configuration is as follows:
|
||||
|
||||
| Parameters | Default value |
|
||||
|
|
@ -381,7 +389,8 @@ The default configuration is as follows:
|
|||
|
||||
### dolphinscheduler_env.sh [load environment variables configs]
|
||||
|
||||
When using shell to commit tasks, DolphinScheduler will export environment variables from `bin/env/dolphinscheduler_env.sh`. The
|
||||
When using shell to commit tasks, DolphinScheduler will export environment variables
|
||||
from `bin/env/dolphinscheduler_env.sh`. The
|
||||
mainly configuration including `JAVA_HOME` and other environment paths.
|
||||
|
||||
```bash
|
||||
|
|
@ -409,9 +418,10 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec
|
|||
|
||||
### Log related configuration
|
||||
|
||||
|Service| Configuration file |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/logback-spring.xml`|
|
||||
|Api Server| `api-server/conf/logback-spring.xml`|
|
||||
|Worker Server| `worker-server/conf/logback-spring.xml`|
|
||||
|Alert Server| `alert-server/conf/logback-spring.xml`|
|
||||
| Service | Configuration file |
|
||||
|---------------|-----------------------------------------|
|
||||
| Master Server | `master-server/conf/logback-spring.xml` |
|
||||
| Api Server | `api-server/conf/logback-spring.xml` |
|
||||
| Worker Server | `worker-server/conf/logback-spring.xml` |
|
||||
| Alert Server | `alert-server/conf/logback-spring.xml` |
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Use the command line mode `cd` enter the `dolphinscheduler-ui` project director
|
|||
> If `npm install` is very slow, you can set the taobao mirror
|
||||
|
||||
```
|
||||
npm config set registry http://registry.npm.taobao.org/
|
||||
npm config set registry http://registry.npmmirror.com/
|
||||
```
|
||||
|
||||
- Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend:
|
||||
|
|
|
|||
|
|
@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory
|
|||
sudo rm -rf node_modules
|
||||
```
|
||||
|
||||
2, install node-sass through npm.taobao.org
|
||||
2, install node-sass through npmmirror.com
|
||||
|
||||
```
|
||||
sudo npm uninstall node-sass
|
||||
sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
|
||||
sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
|
||||
```
|
||||
|
||||
3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md)
|
||||
|
|
|
|||
|
|
@ -7,11 +7,11 @@ The following shows the `Script` configuration example:
|
|||
|
||||
## Parameter Configuration
|
||||
|
||||
| **Parameter** | **Description** |
|
||||
|---------------|--------------------------------------------------|
|
||||
| User Params | User defined parameters will pass to the script. |
|
||||
| Script Path | The file location path in the server. |
|
||||
| Type | Support `Shell` script. |
|
||||
| **Parameter** | **Description** |
|
||||
|---------------|-------------------------------------------------------------|
|
||||
| User Params | User defined parameters will pass to the script. |
|
||||
| Script Path | The file location path in the server, only support .sh file |
|
||||
| Type | Support `Shell` script. |
|
||||
|
||||
### Note
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,15 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
|
|||
|
||||
## Install DolphinScheduler
|
||||
|
||||
Please download the source code package `apache-dolphinscheduler-<version>-src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download)
|
||||
|
||||
To publish the release name `dolphinscheduler` version, please execute the following commands:
|
||||
|
||||
```
|
||||
$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
|
||||
$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler . --set image.tag=<version>
|
||||
```bash
|
||||
# Choose the corresponding version yourself
|
||||
export VERSION=3.2.1
|
||||
helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
|
||||
tar -xvf dolphinscheduler-helm-${VERSION}.tgz
|
||||
cd dolphinscheduler-helm
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm dependency update .
|
||||
helm install dolphinscheduler .
|
||||
```
|
||||
|
||||
To publish the release name `dolphinscheduler` version to `test` namespace:
|
||||
|
|
|
|||
|
|
@ -123,7 +123,6 @@ export SPRING_DATASOURCE_PASSWORD={password}
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ If you deploy DolphinScheduler in `Standalone` mode, you only need to configure
|
|||
```properties
|
||||
# Whether to enable remote logging
|
||||
remote.logging.enable=false
|
||||
# if remote.logging.enable = true, set the target of remote logging
|
||||
# if remote.logging.enable = true, set the target of remote logging, currently support OSS, S3, GCS, ABS
|
||||
remote.logging.target=OSS
|
||||
# if remote.logging.enable = true, set the log base directory
|
||||
remote.logging.base.dir=logs
|
||||
|
|
@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name=<your-bucket>
|
|||
Configure `common.properties` as follows:
|
||||
|
||||
```properties
|
||||
# abs container name, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.container.name=<your-container>
|
||||
# abs account name, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.account.name=<your-account-name>
|
||||
# abs connection string, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.connection.string=<your-connection-string>
|
||||
remote.logging.abs.account.name=<your-account-name>
|
||||
# abs account key, required if you set resource.storage.type=ABS
|
||||
remote.logging.abs.account.key=<your-account-key>
|
||||
# abs container name, required if you set resource.storage.type=ABS
|
||||
remote.logging.abs.container.name=<your-container-name>
|
||||
```
|
||||
|
||||
### Notice
|
||||
|
|
|
|||
|
|
@ -39,3 +39,5 @@ The main configuration parameters are as follows:
|
|||
## Note
|
||||
|
||||
When you run the task in JAVA execution mode, the public class must exist in the code, and you could omit writing a package statement.
|
||||
|
||||
For security reasons, when executing JAVA tasks, please use the environment management module to configure the JDK environment, such as `JAVA_HOME` and other environment variables.
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ Spark task type for executing Spark application. When executing the Spark task,
|
|||
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Program type | Supports Java, Scala, Python, and SQL. |
|
||||
| The class of main function | The **full path** of Main Class, the entry point of the Spark program. |
|
||||
| Master | The The master URL for the cluster. |
|
||||
| Main jar package | The Spark jar package (upload by Resource Center). |
|
||||
| SQL scripts | SQL statements in .sql files that Spark sql runs. |
|
||||
| Deployment mode | <ul><li>spark submit supports three modes: cluster, client and local.</li><li>spark sql supports client and local modes.</li></ul> |
|
||||
|
|
|
|||
|
|
@ -130,38 +130,40 @@ export DOLPHINSCHEDULER_OPTS="
|
|||
|
||||
> 不建议设置"-XX:DisableExplicitGC" , DolphinScheduler使用Netty进行通讯,设置该参数,可能会导致内存泄漏.
|
||||
>
|
||||
>> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true"将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6.
|
||||
>> 如果设置"-Djava.net.preferIPv6Addresses=true" 将会使用ipv6的IP地址， 如果设置"-Djava.net.preferIPv4Addresses=true"
|
||||
>> 将会使用ipv4的IP地址, 如果都不设置，将会随机使用ipv4或者ipv6.
|
||||
|
||||
## 数据库连接相关配置
|
||||
|
||||
在DolphinScheduler中使用Spring Hikari对数据库连接进行管理，配置文件位置：
|
||||
|
||||
|服务名称| 配置文件 |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/application.yaml`|
|
||||
|Api Server| `api-server/conf/application.yaml`|
|
||||
|Worker Server| `worker-server/conf/application.yaml`|
|
||||
|Alert Server| `alert-server/conf/application.yaml`|
|
||||
| 服务名称 | 配置文件 |
|
||||
|---------------|---------------------------------------|
|
||||
| Master Server | `master-server/conf/application.yaml` |
|
||||
| Api Server | `api-server/conf/application.yaml` |
|
||||
| Worker Server | `worker-server/conf/application.yaml` |
|
||||
| Alert Server | `alert-server/conf/application.yaml` |
|
||||
|
||||
默认配置如下：
|
||||
|
||||
|参数 | 默认值| 描述|
|
||||
|--|--|--|
|
||||
|spring.datasource.driver-class-name| org.postgresql.Driver |数据库驱动|
|
||||
|spring.datasource.url| jdbc:postgresql://127.0.0.1:5432/dolphinscheduler |数据库连接地址|
|
||||
|spring.datasource.username|root|数据库用户名|
|
||||
|spring.datasource.password|root|数据库密码|
|
||||
|spring.datasource.hikari.connection-test-query|select 1|检测连接是否有效的sql|
|
||||
|spring.datasource.hikari.minimum-idle| 5|最小空闲连接池数量|
|
||||
|spring.datasource.hikari.auto-commit|true|是否自动提交|
|
||||
|spring.datasource.hikari.pool-name|DolphinScheduler|连接池名称|
|
||||
|spring.datasource.hikari.maximum-pool-size|50|连接池最大连接数|
|
||||
|spring.datasource.hikari.connection-timeout|30000|连接超时时长|
|
||||
|spring.datasource.hikari.idle-timeout|600000|空闲连接存活最大时间|
|
||||
|spring.datasource.hikari.leak-detection-threshold|0|连接泄露检测阈值|
|
||||
|spring.datasource.hikari.initialization-fail-timeout|1|连接池初始化失败timeout|
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|------------------------------------------------------|---------------------------------------------------|-----------------|
|
||||
| spring.datasource.driver-class-name | org.postgresql.Driver | 数据库驱动 |
|
||||
| spring.datasource.url | jdbc:postgresql://127.0.0.1:5432/dolphinscheduler | 数据库连接地址 |
|
||||
| spring.datasource.username | root | 数据库用户名 |
|
||||
| spring.datasource.password | root | 数据库密码 |
|
||||
| spring.datasource.hikari.connection-test-query | select 1 | 检测连接是否有效的sql |
|
||||
| spring.datasource.hikari.minimum-idle | 5 | 最小空闲连接池数量 |
|
||||
| spring.datasource.hikari.auto-commit | true | 是否自动提交 |
|
||||
| spring.datasource.hikari.pool-name | DolphinScheduler | 连接池名称 |
|
||||
| spring.datasource.hikari.maximum-pool-size | 50 | 连接池最大连接数 |
|
||||
| spring.datasource.hikari.connection-timeout | 30000 | 连接超时时长 |
|
||||
| spring.datasource.hikari.idle-timeout | 600000 | 空闲连接存活最大时间 |
|
||||
| spring.datasource.hikari.leak-detection-threshold | 0 | 连接泄露检测阈值 |
|
||||
| spring.datasource.hikari.initialization-fail-timeout | 1 | 连接池初始化失败timeout |
|
||||
|
||||
DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名, 设置值即可。
|
||||
DolphinScheduler同样可以通过设置环境变量进行数据库连接相关的配置, 将以上小写字母转成大写并把`.`换成`_`作为环境变量名,
|
||||
设置值即可。
|
||||
|
||||
## 注册中心相关配置
|
||||
|
||||
|
|
@ -174,17 +176,17 @@ DolphinScheduler默认使用Zookeeper进行集群管理、容错、事件监听
|
|||
|
||||
默认配置如下：
|
||||
|
||||
|参数 |默认值| 描述|
|
||||
|--|--|--|
|
||||
|registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace|
|
||||
|registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息|
|
||||
|registry.zookeeper.retry-policy.base-sleep-time|60ms|基本重试时间差|
|
||||
|registry.zookeeper.retry-policy.max-sleep|300ms|最大重试时间|
|
||||
|registry.zookeeper.retry-policy.max-retries|5|最大重试次数|
|
||||
|registry.zookeeper.session-timeout|30s|session超时时间|
|
||||
|registry.zookeeper.connection-timeout|30s|连接超时时间|
|
||||
|registry.zookeeper.block-until-connected|600ms|阻塞直到连接成功的等待时间|
|
||||
|registry.zookeeper.digest|{用户名:密码}|如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档)|
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-------------------------------------------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| registry.zookeeper.namespace | dolphinscheduler | Zookeeper集群使用的namespace |
|
||||
| registry.zookeeper.connect-string | localhost:2181 | Zookeeper集群连接信息 |
|
||||
| registry.zookeeper.retry-policy.base-sleep-time | 60ms | 基本重试时间差 |
|
||||
| registry.zookeeper.retry-policy.max-sleep | 300ms | 最大重试时间 |
|
||||
| registry.zookeeper.retry-policy.max-retries | 5 | 最大重试次数 |
|
||||
| registry.zookeeper.session-timeout | 30s | session超时时间 |
|
||||
| registry.zookeeper.connection-timeout | 30s | 连接超时时间 |
|
||||
| registry.zookeeper.block-until-connected | 600ms | 阻塞直到连接成功的等待时间 |
|
||||
| registry.zookeeper.digest | {用户名:密码} | 如果zookeeper打开了acl，则需要填写认证信息访问znode，认证信息格式为{用户名}:{密码}。关于Zookeeper ACL详见[https://zookeeper.apache.org/doc/r3.4.14/zookeeperAdmin.html](Apache Zookeeper官方文档) |
|
||||
|
||||
DolphinScheduler同样可以通过`bin/env/dolphinscheduler_env.sh`进行Zookeeper相关的配置。
|
||||
|
||||
|
|
@ -203,8 +205,8 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
|
||||
默认配置如下：
|
||||
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------|--------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| data.basedir.path | /tmp/dolphinscheduler | 本地工作目录,用于存放临时文件 |
|
||||
| resource.storage.type | NONE | 资源文件存储类型: HDFS,S3,OSS,GCS,ABS,NONE |
|
||||
| resource.upload.path | /dolphinscheduler | 资源文件存储路径 |
|
||||
|
|
@ -282,48 +284,55 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
|
||||
位置：`master-server/conf/application.yaml`
|
||||
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|--------------------------------------------------------|--------------|-----------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master监听端口 |
|
||||
| master.fetch-command-num | 10 | master拉取command数量 |
|
||||
| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command |
|
||||
| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 |
|
||||
| master.dispatch-task-number | 3 | master每个批次的派发任务数量 |
|
||||
| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight |
|
||||
| master.max-heartbeat-interval | 10s | master最大心跳间隔 |
|
||||
| master.task-commit-retry-times | 5 | 任务重试次数 |
|
||||
| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 |
|
||||
| master.state-wheel-interval | 5 | 轮询检查状态时间 |
|
||||
| master.max-cpu-load-avg | 1 | master最大cpuload均值,只有高于系统cpuload均值时,master服务才能调度任务. 默认值为1: 会使用100%的CPU |
|
||||
| master.reserved-memory | 0.3 | master预留内存,只有低于系统可用内存时,master服务才能调度任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流 |
|
||||
| master.failover-interval | 10 | failover间隔，单位为分钟 |
|
||||
| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application |
|
||||
| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, |
|
||||
| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
|
||||
| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 |
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master监听端口 |
|
||||
| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command |
|
||||
| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 |
|
||||
| master.dispatch-task-number | 3 | master每个批次的派发任务数量 |
|
||||
| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight |
|
||||
| master.max-heartbeat-interval | 10s | master最大心跳间隔 |
|
||||
| master.task-commit-retry-times | 5 | 任务重试次数 |
|
||||
| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 |
|
||||
| master.state-wheel-interval | 5 | 轮询检查状态时间 |
|
||||
| master.server-load-protection.enabled | true | 是否开启系统保护策略 |
|
||||
| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU |
|
||||
| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU |
|
||||
| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 |
|
||||
| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
|
||||
| master.failover-interval | 10 | failover间隔，单位为分钟 |
|
||||
| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application |
|
||||
| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
|
||||
| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 |
|
||||
| master.command-fetch-strategy.type | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED` |
|
||||
| master.command-fetch-strategy.config.id-step | 1 | 数据库中t_ds_command的id自增步长 |
|
||||
| master.command-fetch-strategy.config.fetch-size | 10 | master拉取command数量 |
|
||||
|
||||
## Worker Server相关配置
|
||||
|
||||
位置：`worker-server/conf/application.yaml`
|
||||
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| worker.listen-port | 1234 | worker监听端口 |
|
||||
| worker.exec-threads | 100 | worker工作线程数量,用于限制并行的任务实例数量 |
|
||||
| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 |
|
||||
| worker.host-weight | 100 | 派发任务时，worker主机的权重 |
|
||||
| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
|
||||
| worker.max-cpu-load-avg | 1 | worker最大cpuload均值,只有高于系统cpuload均值时,worker服务才能被派发任务. 默认值为1: 会使用100%的CPU |
|
||||
| worker.reserved-memory | 0.3 | worker预留内存,只有低于系统可用内存时,worker服务才能被派发任务. 默认值为0.3：当系统内存低于30%时会停止调度新的工作流 |
|
||||
| worker.alert-listen-host | localhost | alert监听host |
|
||||
| worker.alert-listen-port | 50052 | alert监听端口 |
|
||||
| worker.registry-disconnect-strategy.strategy | stop | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
|
||||
| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 |
|
||||
| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| worker.listen-port | 1234 | worker监听端口 |
|
||||
| worker.exec-threads | 100 | worker工作线程数量,用于限制并行的任务实例数量 |
|
||||
| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 |
|
||||
| worker.host-weight | 100 | 派发任务时，worker主机的权重 |
|
||||
| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
|
||||
| worker.server-load-protection.enabled | true | 是否开启系统保护策略 |
|
||||
| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | worker最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统CPU |
|
||||
| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | worker最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的JVM CPU |
|
||||
| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | worker最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统内存 |
|
||||
| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | worker最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
|
||||
| worker.alert-listen-host | localhost | alert监听host |
|
||||
| worker.alert-listen-port | 50052 | alert监听端口 |
|
||||
| worker.registry-disconnect-strategy.strategy | stop | 当Worker与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
|
||||
| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 |
|
||||
| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
|
||||
|
||||
## Alert Server相关配置
|
||||
|
||||
|
|
@ -369,7 +378,9 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
| spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 |
|
||||
| spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool |
|
||||
|
||||
因为*Api Server*不会启动*Quartz Scheduler*实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`不会启动任何线程。具体的默认配置如下：
|
||||
因为*Api Server*不会启动*Quartz Scheduler*
|
||||
实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`
|
||||
不会启动任何线程。具体的默认配置如下：
|
||||
|
||||
| Parameters | Default value |
|
||||
|------------------------------------------------------|-----------------------------------------------------------------------|
|
||||
|
|
@ -377,7 +388,8 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
|
||||
## dolphinscheduler_env.sh [环境变量配置]
|
||||
|
||||
通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME` 任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。
|
||||
通过类似shell方式提交任务的的时候，会加载该配置文件中的环境变量到主机中。涉及到的 `JAVA_HOME`
|
||||
任务类型的环境配置，其中任务类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等。
|
||||
|
||||
```bash
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
|
|
@ -404,9 +416,10 @@ export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspec
|
|||
|
||||
## 日志相关配置
|
||||
|
||||
|服务名称| 配置文件 |
|
||||
|--|--|
|
||||
|Master Server | `master-server/conf/logback-spring.xml`|
|
||||
|Api Server| `api-server/conf/logback-spring.xml`|
|
||||
|Worker Server| `worker-server/conf/logback-spring.xml`|
|
||||
|Alert Server| `alert-server/conf/logback-spring.xml`|
|
||||
| 服务名称 | 配置文件 |
|
||||
|---------------|-----------------------------------------|
|
||||
| Master Server | `master-server/conf/logback-spring.xml` |
|
||||
| Api Server | `api-server/conf/logback-spring.xml` |
|
||||
| Worker Server | `worker-server/conf/logback-spring.xml` |
|
||||
| Alert Server | `alert-server/conf/logback-spring.xml` |
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2
|
|||
> 如果 `npm install` 速度非常慢，你可以设置淘宝镜像
|
||||
|
||||
```
|
||||
npm config set registry http://registry.npm.taobao.org/
|
||||
npm config set registry http://registry.npmmirror.com/
|
||||
```
|
||||
|
||||
- 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互：
|
||||
|
|
|
|||
|
|
@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录
|
|||
sudo rm -rf node_modules
|
||||
```
|
||||
|
||||
2，通过 npm.taobao.org 下载 node-sass
|
||||
2，通过 npmmirror.com 下载 node-sass
|
||||
|
||||
```
|
||||
sudo npm uninstall node-sass
|
||||
sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
|
||||
sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
|
||||
```
|
||||
|
||||
3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
|
||||
* 脚本路径
|
||||
|
||||
> 脚本在服务器上的文件位置
|
||||
> 脚本在服务器上的文件位置，只支持.sh后缀的文件
|
||||
|
||||
* 脚本类型
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,15 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务
|
|||
|
||||
## 安装 dolphinscheduler
|
||||
|
||||
请下载源码包 apache-dolphinscheduler-<version>-src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download)
|
||||
|
||||
发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令：
|
||||
|
||||
```
|
||||
$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
|
||||
$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler . --set image.tag=<version>
|
||||
```bash
|
||||
# 自行选择对应的版本
|
||||
export VERSION=3.2.1
|
||||
helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
|
||||
tar -xvf dolphinscheduler-helm-${VERSION}.tgz
|
||||
cd dolphinscheduler-helm
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm dependency update .
|
||||
helm install dolphinscheduler .
|
||||
```
|
||||
|
||||
将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中：
|
||||
|
|
|
|||
|
|
@ -118,7 +118,6 @@ export SPRING_DATASOURCE_PASSWORD={password}
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Apache DolphinScheduler支持将任务日志传输到远端存储上。当配置
|
|||
```properties
|
||||
# 是否开启远程日志存储
|
||||
remote.logging.enable=true
|
||||
# 任务日志写入的远端存储，目前支持OSS, S3, GCS
|
||||
# 任务日志写入的远端存储，目前支持OSS, S3, GCS, ABS
|
||||
remote.logging.target=OSS
|
||||
# 任务日志在远端存储上的目录
|
||||
remote.logging.base.dir=logs
|
||||
|
|
@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name=<your-bucket>
|
|||
配置`common.propertis`如下：
|
||||
|
||||
```properties
|
||||
# abs container name, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.container.name=<your-container>
|
||||
# abs account name, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.account.name=<your-account-name>
|
||||
# abs connection string, required if you set resource.storage.type=ABS
|
||||
resource.azure.blob.storage.connection.string=<your-connection-string>
|
||||
remote.logging.abs.account.name=<your-account-name>
|
||||
# abs account key, required if you set resource.storage.type=ABS
|
||||
remote.logging.abs.account.key=<your-account-key>
|
||||
# abs container name, required if you set resource.storage.type=ABS
|
||||
remote.logging.abs.container.name=<your-container-name>
|
||||
```
|
||||
|
||||
### 注意事项
|
||||
|
|
|
|||
|
|
@ -43,3 +43,4 @@ java任务类型有两种运行模式，这里以JAVA模式为例进行演示。
|
|||
|
||||
使用JAVA运行类型时代码中必须存在public类，可以不写package语句
|
||||
|
||||
基于安全原因，执行JAVA任务时，请使用环境管理功能配置JDK环境，例如`JAVA_HOME`等环境变量
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ Spark 任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支
|
|||
- 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。
|
||||
- 主函数的 Class：Spark 程序的入口 Main class 的全路径。
|
||||
- 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。
|
||||
- Master：执行 Spark 集群的 Master Url。
|
||||
- SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。
|
||||
- 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。
|
||||
(2) spark sql 支持 client 和 local 两种模式。
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ public final class VoiceAlertChannel implements AlertChannel {
|
|||
|
||||
Map<String, String> paramsMap = info.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "aliyun-voice params is null");
|
||||
return new AlertResult(false, "aliyun-voice params is null");
|
||||
}
|
||||
VoiceParam voiceParam = buildVoiceParam(paramsMap);
|
||||
return new VoiceSender(voiceParam).send();
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public final class VoiceSender {
|
|||
|
||||
public AlertResult send() {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
try {
|
||||
Client client = createClient(voiceParam.getConnection());
|
||||
SingleCallByTtsRequest singleCallByTtsRequest = new SingleCallByTtsRequest()
|
||||
|
|
@ -61,7 +61,7 @@ public final class VoiceSender {
|
|||
}
|
||||
SingleCallByTtsResponseBody body = response.getBody();
|
||||
if (body.code.equalsIgnoreCase("ok")) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage(body.getCallId());
|
||||
} else {
|
||||
alertResult.setMessage(body.getMessage());
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class VoiceSenderTest {
|
|||
VoiceSender weChatSender = new VoiceSender(voiceParam);
|
||||
|
||||
AlertResult alertResult = weChatSender.send();
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,6 @@ public interface AlertChannel {
|
|||
AlertResult process(AlertInfo info);
|
||||
|
||||
default @NonNull AlertResult closeAlert(AlertInfo info) {
|
||||
return new AlertResult("true", "no need to close alert");
|
||||
return new AlertResult(true, "no need to close alert");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,14 +53,6 @@ public class AlertData {
|
|||
*/
|
||||
private String log;
|
||||
|
||||
/**
|
||||
* 0 do not send warning;
|
||||
* 1 send if process success;
|
||||
* 2 send if process failed;
|
||||
* 3 send if process ends, whatever the result;
|
||||
*/
|
||||
private int warnType;
|
||||
|
||||
/**
|
||||
* AlertType#code
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -33,15 +33,19 @@ import lombok.NoArgsConstructor;
|
|||
@NoArgsConstructor
|
||||
public class AlertResult {
|
||||
|
||||
/**
|
||||
* todo: use enum
|
||||
* false or true
|
||||
*/
|
||||
private String status;
|
||||
private boolean success;
|
||||
|
||||
/**
|
||||
* alert result message, each plugin can have its own message
|
||||
*/
|
||||
private String message;
|
||||
|
||||
public static AlertResult success() {
|
||||
return new AlertResult(true, null);
|
||||
}
|
||||
|
||||
public static AlertResult fail(String message) {
|
||||
return new AlertResult(false, message);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public final class DingTalkAlertChannel implements AlertChannel {
|
|||
AlertData alertData = alertInfo.getAlertData();
|
||||
Map<String, String> paramsMap = alertInfo.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "ding talk params is null");
|
||||
return new AlertResult(false, "ding talk params is null");
|
||||
}
|
||||
return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ import java.util.Objects;
|
|||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
|
|
@ -124,7 +126,7 @@ public final class DingTalkSender {
|
|||
|
||||
private AlertResult checkSendDingTalkSendMsgResult(String result) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
|
||||
if (null == result) {
|
||||
alertResult.setMessage("send ding talk msg error");
|
||||
|
|
@ -138,7 +140,7 @@ public final class DingTalkSender {
|
|||
return alertResult;
|
||||
}
|
||||
if (sendMsgResponse.errcode == 0) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage("send ding talk msg success");
|
||||
return alertResult;
|
||||
}
|
||||
|
|
@ -162,7 +164,7 @@ public final class DingTalkSender {
|
|||
} catch (Exception e) {
|
||||
log.info("send ding talk alert msg exception : {}", e.getMessage());
|
||||
alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage("send ding talk alert fail.");
|
||||
}
|
||||
return alertResult;
|
||||
|
|
@ -189,7 +191,7 @@ public final class DingTalkSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "UTF-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
@ -317,15 +319,17 @@ public final class DingTalkSender {
|
|||
String sign = org.apache.commons.lang3.StringUtils.EMPTY;
|
||||
try {
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256"));
|
||||
byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8"));
|
||||
sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8");
|
||||
mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
|
||||
byte[] signData = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8));
|
||||
sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), StandardCharsets.UTF_8.name());
|
||||
} catch (Exception e) {
|
||||
log.error("generate sign error, message:{}", e);
|
||||
}
|
||||
return url + "×tamp=" + timestamp + "&sign=" + sign;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
static final class DingTalkSendMsgResponse {
|
||||
|
||||
private Integer errcode;
|
||||
|
|
@ -334,22 +338,6 @@ public final class DingTalkSender {
|
|||
public DingTalkSendMsgResponse() {
|
||||
}
|
||||
|
||||
public Integer getErrcode() {
|
||||
return this.errcode;
|
||||
}
|
||||
|
||||
public void setErrcode(Integer errcode) {
|
||||
this.errcode = errcode;
|
||||
}
|
||||
|
||||
public String getErrmsg() {
|
||||
return this.errmsg;
|
||||
}
|
||||
|
||||
public void setErrmsg(String errmsg) {
|
||||
this.errmsg = errmsg;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o) {
|
||||
if (o == this) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk;
|
|||
|
||||
import org.apache.dolphinscheduler.alert.api.AlertResult;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -47,11 +48,11 @@ public class DingTalkSenderTest {
|
|||
@Test
|
||||
public void testSend() {
|
||||
DingTalkSender dingTalkSender = new DingTalkSender(dingTalkConfig);
|
||||
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8");
|
||||
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", StandardCharsets.UTF_8.name());
|
||||
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true");
|
||||
dingTalkSender = new DingTalkSender(dingTalkConfig);
|
||||
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertEquals(false, alertResult.isSuccess());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,24 +35,20 @@ public final class EmailAlertChannel implements AlertChannel {
|
|||
AlertData alert = info.getAlertData();
|
||||
Map<String, String> paramsMap = info.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "mail params is null");
|
||||
return new AlertResult(false, "mail params is null");
|
||||
}
|
||||
MailSender mailSender = new MailSender(paramsMap);
|
||||
AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent());
|
||||
|
||||
boolean flag;
|
||||
|
||||
if (alertResult == null) {
|
||||
alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage("alert send error.");
|
||||
log.info("alert send error : {}", alertResult.getMessage());
|
||||
return alertResult;
|
||||
}
|
||||
|
||||
flag = Boolean.parseBoolean(String.valueOf(alertResult.getStatus()));
|
||||
|
||||
if (flag) {
|
||||
if (alertResult.isSuccess()) {
|
||||
log.info("alert send success");
|
||||
alertResult.setMessage("email send success.");
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -56,8 +56,6 @@ public final class EmailConstants {
|
|||
|
||||
public static final String TABLE_BODY_HTML_TAIL = "</table></body></html>";
|
||||
|
||||
public static final String UTF_8 = "UTF-8";
|
||||
|
||||
public static final String EXCEL_SUFFIX_XLSX = ".xlsx";
|
||||
|
||||
public static final String SINGLE_SLASH = "/";
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.commons.mail.HtmlEmail;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
|
@ -153,7 +154,7 @@ public final class MailSender {
|
|||
*/
|
||||
public AlertResult sendMails(List<String> receivers, List<String> receiverCcs, String title, String content) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
|
||||
// if there is no receivers && no receiversCc, no need to process
|
||||
if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiverCcs)) {
|
||||
|
|
@ -171,7 +172,7 @@ public final class MailSender {
|
|||
Session session = getSession();
|
||||
email.setMailSession(session);
|
||||
email.setFrom(mailSenderEmail);
|
||||
email.setCharset(EmailConstants.UTF_8);
|
||||
email.setCharset(StandardCharsets.UTF_8.name());
|
||||
if (CollectionUtils.isNotEmpty(receivers)) {
|
||||
// receivers mail
|
||||
for (String receiver : receivers) {
|
||||
|
|
@ -200,7 +201,7 @@ public final class MailSender {
|
|||
|
||||
attachment(title, content, partContent);
|
||||
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
return alertResult;
|
||||
} catch (Exception e) {
|
||||
handleException(alertResult, e);
|
||||
|
|
@ -344,7 +345,8 @@ public final class MailSender {
|
|||
ExcelUtils.genExcelFile(content, randomFilename, xlsFilePath);
|
||||
|
||||
part2.attachFile(file);
|
||||
part2.setFileName(MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, EmailConstants.UTF_8, "B"));
|
||||
part2.setFileName(
|
||||
MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, StandardCharsets.UTF_8.name(), "B"));
|
||||
// add components to collection
|
||||
partList.addBodyPart(part1);
|
||||
partList.addBodyPart(part2);
|
||||
|
|
@ -378,7 +380,7 @@ public final class MailSender {
|
|||
email.setDebug(true);
|
||||
email.send();
|
||||
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
|
||||
return alertResult;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ public class EmailAlertChannelTest {
|
|||
alertInfo.setAlertParams(paramsMap);
|
||||
AlertResult alertResult = emailAlertChannel.process(alertInfo);
|
||||
Assertions.assertNotNull(alertResult);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
public String getEmailAlertParams() {
|
||||
|
|
|
|||
|
|
@ -66,8 +66,15 @@ public class ExcelUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testGenExcelFileByCheckDir() {
|
||||
ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls");
|
||||
File file = new File("/tmp/xls" + EmailConstants.SINGLE_SLASH + "t" + EmailConstants.EXCEL_SUFFIX_XLSX);
|
||||
String path = "/tmp/xls";
|
||||
ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", path);
|
||||
File file =
|
||||
new File(
|
||||
path
|
||||
+ EmailConstants.SINGLE_SLASH
|
||||
+ "t"
|
||||
+ EmailConstants.EXCEL_SUFFIX_XLSX);
|
||||
file.delete();
|
||||
Assertions.assertFalse(file.exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,9 @@
|
|||
package org.apache.dolphinscheduler.plugin.alert.email;
|
||||
|
||||
import org.apache.dolphinscheduler.alert.api.AlertConstants;
|
||||
import org.apache.dolphinscheduler.alert.api.AlertResult;
|
||||
import org.apache.dolphinscheduler.alert.api.ShowType;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate;
|
||||
import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -42,7 +41,6 @@ public class MailUtilsTest {
|
|||
private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class);
|
||||
static MailSender mailSender;
|
||||
private static Map<String, String> emailConfig = new HashMap<>();
|
||||
private static AlertTemplate alertTemplate;
|
||||
|
||||
@BeforeAll
|
||||
public static void initEmailConfig() {
|
||||
|
|
@ -59,7 +57,6 @@ public class MailUtilsTest {
|
|||
emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, "347801120@qq.com");
|
||||
emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, "347801120@qq.com");
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp());
|
||||
alertTemplate = new DefaultHTMLTemplate();
|
||||
mailSender = new MailSender(emailConfig);
|
||||
}
|
||||
|
||||
|
|
@ -77,9 +74,10 @@ public class MailUtilsTest {
|
|||
+ "\"Host: 192.168.xx.xx\","
|
||||
+ "\"Notify group :4\"]";
|
||||
|
||||
mailSender.sendMails(
|
||||
AlertResult alertResult = mailSender.sendMails(
|
||||
"Mysql Exception",
|
||||
content);
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -108,7 +106,8 @@ public class MailUtilsTest {
|
|||
emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "user");
|
||||
emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd");
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails(title, content);
|
||||
AlertResult alertResult = mailSender.sendMails(title, content);
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
public String list2String() {
|
||||
|
|
@ -134,7 +133,6 @@ public class MailUtilsTest {
|
|||
logger.info(mapjson);
|
||||
|
||||
return mapjson;
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -143,23 +141,25 @@ public class MailUtilsTest {
|
|||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails(title, content);
|
||||
AlertResult alertResult = mailSender.sendMails(title, content);
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAttachmentFile() throws Exception {
|
||||
public void testAttachmentFile() {
|
||||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails("gaojing", content);
|
||||
AlertResult alertResult = mailSender.sendMails("gaojing", content);
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTableAttachmentFile() throws Exception {
|
||||
public void testTableAttachmentFile() {
|
||||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails("gaojing", content);
|
||||
AlertResult alertResult = mailSender.sendMails("gaojing", content);
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public final class FeiShuAlertChannel implements AlertChannel {
|
|||
AlertData alertData = alertInfo.getAlertData();
|
||||
Map<String, String> paramsMap = alertInfo.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "fei shu params is null");
|
||||
return new AlertResult(false, "fei shu params is null");
|
||||
}
|
||||
return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -79,7 +80,7 @@ public final class FeiShuSender {
|
|||
|
||||
public static AlertResult checkSendFeiShuSendMsgResult(String result) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
|
||||
if (org.apache.commons.lang3.StringUtils.isBlank(result)) {
|
||||
alertResult.setMessage("send fei shu msg error");
|
||||
|
|
@ -94,7 +95,7 @@ public final class FeiShuSender {
|
|||
return alertResult;
|
||||
}
|
||||
if (sendMsgResponse.statusCode == 0) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage("send fei shu msg success");
|
||||
return alertResult;
|
||||
}
|
||||
|
|
@ -135,7 +136,7 @@ public final class FeiShuSender {
|
|||
} catch (Exception e) {
|
||||
log.info("send fei shu alert msg exception : {}", e.getMessage());
|
||||
alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage("send fei shu alert fail.");
|
||||
}
|
||||
return alertResult;
|
||||
|
|
@ -161,7 +162,7 @@ public final class FeiShuSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "utf-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ public class FeiShuSenderTest {
|
|||
alertData.setContent("feishu test content");
|
||||
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
|
||||
AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -87,12 +87,12 @@ public class FeiShuSenderTest {
|
|||
|
||||
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
|
||||
AlertResult alertResult = feiShuSender.checkSendFeiShuSendMsgResult("");
|
||||
Assertions.assertFalse(Boolean.valueOf(alertResult.getStatus()));
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
AlertResult alertResult2 = feiShuSender.checkSendFeiShuSendMsgResult("123");
|
||||
Assertions.assertEquals("send fei shu msg fail", alertResult2.getMessage());
|
||||
|
||||
String response = "{\"StatusCode\":\"0\",\"extra\":\"extra\",\"StatusMessage\":\"StatusMessage\"}";
|
||||
AlertResult alertResult3 = feiShuSender.checkSendFeiShuSendMsgResult(response);
|
||||
Assertions.assertTrue(Boolean.valueOf(alertResult3.getStatus()));
|
||||
Assertions.assertTrue(alertResult3.isSuccess());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public final class HttpAlertChannel implements AlertChannel {
|
|||
AlertData alertData = alertInfo.getAlertData();
|
||||
Map<String, String> paramsMap = alertInfo.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "http params is null");
|
||||
return new AlertResult(false, "http params is null");
|
||||
}
|
||||
|
||||
return new HttpSender(paramsMap).send(alertData.getContent());
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import java.net.URI;
|
|||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -58,7 +59,6 @@ public final class HttpSender {
|
|||
* request type get
|
||||
*/
|
||||
private static final String REQUEST_TYPE_GET = "GET";
|
||||
private static final String DEFAULT_CHARSET = "utf-8";
|
||||
private final String headerParams;
|
||||
private final String bodyParams;
|
||||
private final String contentField;
|
||||
|
|
@ -92,18 +92,18 @@ public final class HttpSender {
|
|||
}
|
||||
|
||||
if (httpRequest == null) {
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage("Request types are not supported");
|
||||
return alertResult;
|
||||
}
|
||||
|
||||
try {
|
||||
String resp = this.getResponseString(httpRequest);
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage(resp);
|
||||
} catch (Exception e) {
|
||||
log.error("send http alert msg exception : {}", e.getMessage());
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage(
|
||||
String.format("Send http request alert failed: %s", e.getMessage()));
|
||||
}
|
||||
|
|
@ -124,7 +124,7 @@ public final class HttpSender {
|
|||
|
||||
CloseableHttpResponse response = httpClient.execute(httpRequest);
|
||||
HttpEntity entity = response.getEntity();
|
||||
return EntityUtils.toString(entity, DEFAULT_CHARSET);
|
||||
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private void createHttpRequest(String msg) throws MalformedURLException, URISyntaxException {
|
||||
|
|
@ -157,7 +157,8 @@ public final class HttpSender {
|
|||
type = URL_SPLICE_CHAR;
|
||||
}
|
||||
try {
|
||||
url = String.format("%s%s%s=%s", url, type, contentField, URLEncoder.encode(msg, DEFAULT_CHARSET));
|
||||
url = String.format("%s%s%s=%s", url, type, contentField,
|
||||
URLEncoder.encode(msg, StandardCharsets.UTF_8.name()));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -190,7 +191,7 @@ public final class HttpSender {
|
|||
}
|
||||
// set msg content field
|
||||
objectNode.put(contentField, msg);
|
||||
StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), DEFAULT_CHARSET);
|
||||
StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), StandardCharsets.UTF_8);
|
||||
((HttpPost) httpRequest).setEntity(entity);
|
||||
} catch (Exception e) {
|
||||
log.error("send http alert msg exception : {}", e.getMessage());
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ public class HttpAlertChannelTest {
|
|||
|
||||
// HttpSender(paramsMap).send(alertData.getContent()); already test in HttpSenderTest.sendTest. so we can mock
|
||||
// it
|
||||
doReturn(new AlertResult("true", "success")).when(alertChannel).process(any());
|
||||
doReturn(new AlertResult(true, "success")).when(alertChannel).process(any());
|
||||
AlertResult alertResult = alertChannel.process(alertInfo);
|
||||
Assertions.assertEquals("true", alertResult.getStatus());
|
||||
Assertions.assertTrue(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public class HttpSenderTest {
|
|||
HttpSender httpSender = spy(new HttpSender(paramsMap));
|
||||
doReturn("success").when(httpSender).getResponseString(any());
|
||||
AlertResult alertResult = httpSender.send("Fault tolerance warning");
|
||||
Assertions.assertEquals("true", alertResult.getStatus());
|
||||
Assertions.assertTrue(alertResult.isSuccess());
|
||||
Assertions.assertTrue(httpSender.getRequestUrl().contains(url));
|
||||
Assertions.assertTrue(httpSender.getRequestUrl().contains(contentField));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ public final class PagerDutyAlertChannel implements AlertChannel {
|
|||
public AlertResult process(AlertInfo alertInfo) {
|
||||
AlertData alertData = alertInfo.getAlertData();
|
||||
Map<String, String> alertParams = alertInfo.getAlertParams();
|
||||
if (alertParams == null || alertParams.size() == 0) {
|
||||
return new AlertResult("false", "PagerDuty alert params is empty");
|
||||
if (alertParams == null || alertParams.isEmpty()) {
|
||||
return new AlertResult(false, "PagerDuty alert params is empty");
|
||||
}
|
||||
|
||||
return new PagerDutySender(alertParams).sendPagerDutyAlter(alertData.getTitle(), alertData.getContent());
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ public final class PagerDutySender {
|
|||
|
||||
public AlertResult sendPagerDutyAlter(String title, String content) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage("send pager duty alert fail.");
|
||||
|
||||
try {
|
||||
|
|
@ -83,7 +83,7 @@ public final class PagerDutySender {
|
|||
String responseContent = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
try {
|
||||
if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage("send pager duty alert success");
|
||||
} else {
|
||||
alertResult.setMessage(
|
||||
|
|
|
|||
|
|
@ -39,6 +39,6 @@ public class PagerDutySenderTest {
|
|||
public void testSend() {
|
||||
PagerDutySender pagerDutySender = new PagerDutySender(pagerDutyConfig);
|
||||
AlertResult alertResult = pagerDutySender.sendPagerDutyAlter("pagerduty test title", "pagerduty test content");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public final class PrometheusAlertChannel implements AlertChannel {
|
|||
AlertData alertData = info.getAlertData();
|
||||
Map<String, String> paramsMap = info.getAlertParams();
|
||||
if (null == paramsMap) {
|
||||
return new AlertResult("false", "prometheus alert manager params is null");
|
||||
return new AlertResult(false, "prometheus alert manager params is null");
|
||||
}
|
||||
return new PrometheusAlertSender(paramsMap).sendMessage(alertData);
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.alert.api.HttpServiceRetryStrategy;
|
|||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
|
|
@ -34,6 +35,7 @@ import org.apache.http.impl.client.HttpClients;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
@ -63,11 +65,10 @@ public class PrometheusAlertSender {
|
|||
String resp = sendMsg(alertData);
|
||||
return checkSendAlertManageMsgResult(resp);
|
||||
} catch (Exception e) {
|
||||
String errorMsg = String.format("send prometheus alert manager alert error, exception: %s", e.getMessage());
|
||||
log.error(errorMsg);
|
||||
log.error("Send prometheus alert manager alert error", e);
|
||||
alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setMessage(errorMsg);
|
||||
alertResult.setSuccess(false);
|
||||
alertResult.setMessage(ExceptionUtils.getMessage(e));
|
||||
}
|
||||
return alertResult;
|
||||
}
|
||||
|
|
@ -90,7 +91,7 @@ public class PrometheusAlertSender {
|
|||
}
|
||||
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "utf-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
log.error(
|
||||
"Prometheus alert manager send alert failed, http status code: {}, title: {} ,content: {}, resp: {}",
|
||||
|
|
@ -105,10 +106,10 @@ public class PrometheusAlertSender {
|
|||
|
||||
public AlertResult checkSendAlertManageMsgResult(String resp) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
|
||||
if (Objects.equals(resp, PrometheusAlertConstants.ALERT_SUCCESS)) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage("prometheus alert manager send success");
|
||||
return alertResult;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,17 +55,17 @@ public class PrometheusAlertSenderTest {
|
|||
" }]");
|
||||
PrometheusAlertSender sender = new PrometheusAlertSender(config);
|
||||
AlertResult result = sender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckSendAlertManageMsgResult() {
|
||||
PrometheusAlertSender prometheusAlertSender = new PrometheusAlertSender(config);
|
||||
AlertResult alertResult1 = prometheusAlertSender.checkSendAlertManageMsgResult("");
|
||||
Assertions.assertFalse(Boolean.parseBoolean(alertResult1.getStatus()));
|
||||
Assertions.assertFalse(alertResult1.isSuccess());
|
||||
Assertions.assertEquals("prometheus alert manager send fail, resp is ", alertResult1.getMessage());
|
||||
AlertResult alertResult2 = prometheusAlertSender.checkSendAlertManageMsgResult("alert success");
|
||||
Assertions.assertTrue(Boolean.parseBoolean(alertResult2.getStatus()));
|
||||
Assertions.assertTrue(alertResult2.isSuccess());
|
||||
Assertions.assertEquals("prometheus alert manager send success", alertResult2.getMessage());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ public final class ScriptAlertChannel implements AlertChannel {
|
|||
AlertData alertData = alertinfo.getAlertData();
|
||||
Map<String, String> paramsMap = alertinfo.getAlertParams();
|
||||
if (MapUtils.isEmpty(paramsMap)) {
|
||||
return new AlertResult("false", "script params is empty");
|
||||
return new AlertResult(false, "script params is empty");
|
||||
}
|
||||
return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle(), alertData.getContent());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ public final class ScriptSender {
|
|||
}
|
||||
// If it is another type of alarm script can be added here, such as python
|
||||
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
log.error("script type error: {}", scriptType);
|
||||
alertResult.setMessage("script type error : " + scriptType);
|
||||
return alertResult;
|
||||
|
|
@ -64,11 +64,16 @@ public final class ScriptSender {
|
|||
|
||||
private AlertResult executeShellScript(String title, String content) {
|
||||
AlertResult alertResult = new AlertResult();
|
||||
alertResult.setStatus("false");
|
||||
alertResult.setSuccess(false);
|
||||
if (Boolean.TRUE.equals(OSUtils.isWindows())) {
|
||||
alertResult.setMessage("shell script not support windows os");
|
||||
return alertResult;
|
||||
}
|
||||
if (!scriptPath.endsWith(".sh")) {
|
||||
alertResult.setMessage("shell script is invalid, only support .sh file");
|
||||
return alertResult;
|
||||
}
|
||||
|
||||
// validate script path in case of injections
|
||||
File shellScriptFile = new File(scriptPath);
|
||||
// validate existence
|
||||
|
|
@ -106,7 +111,7 @@ public final class ScriptSender {
|
|||
int exitCode = ProcessUtils.executeScript(cmd);
|
||||
|
||||
if (exitCode == 0) {
|
||||
alertResult.setStatus("true");
|
||||
alertResult.setSuccess(true);
|
||||
alertResult.setMessage("send script alert msg success");
|
||||
return alertResult;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public class ProcessUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testExecuteScript() {
|
||||
ProcessUtils.executeScript(cmd);
|
||||
int code = ProcessUtils.executeScript(cmd);
|
||||
assert code != -1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
package org.apache.dolphinscheduler.plugin.alert.script;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
import org.apache.dolphinscheduler.alert.api.AlertResult;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
|
@ -48,9 +50,9 @@ public class ScriptSenderTest {
|
|||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult;
|
||||
alertResult = scriptSender.sendScriptAlert("test title Kris", "test content");
|
||||
Assertions.assertEquals("true", alertResult.getStatus());
|
||||
Assertions.assertTrue(alertResult.isSuccess());
|
||||
alertResult = scriptSender.sendScriptAlert("error msg title", "test content");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -58,7 +60,7 @@ public class ScriptSenderTest {
|
|||
scriptConfig.put(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, "' ; calc.exe ; '");
|
||||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult = scriptSender.sendScriptAlert("test title Kris", "test content");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -67,7 +69,7 @@ public class ScriptSenderTest {
|
|||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult;
|
||||
alertResult = scriptSender.sendScriptAlert("test user params NPE", "test content");
|
||||
Assertions.assertEquals("true", alertResult.getStatus());
|
||||
Assertions.assertTrue(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -76,7 +78,17 @@ public class ScriptSenderTest {
|
|||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult;
|
||||
alertResult = scriptSender.sendScriptAlert("test path NPE", "test content");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
Assertions.assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPathError() {
|
||||
scriptConfig.put(ScriptParamsConstants.NAME_SCRIPT_PATH, "/usr/sbin/abc");
|
||||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult;
|
||||
alertResult = scriptSender.sendScriptAlert("test path NPE", "test content");
|
||||
assertFalse(alertResult.isSuccess());
|
||||
Assertions.assertTrue(alertResult.getMessage().contains("shell script is invalid, only support .sh file"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -85,7 +97,7 @@ public class ScriptSenderTest {
|
|||
ScriptSender scriptSender = new ScriptSender(scriptConfig);
|
||||
AlertResult alertResult;
|
||||
alertResult = scriptSender.sendScriptAlert("test type is error", "test content");
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
assertFalse(alertResult.isSuccess());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,11 +30,11 @@ public final class SlackAlertChannel implements AlertChannel {
|
|||
public AlertResult process(AlertInfo alertInfo) {
|
||||
AlertData alertData = alertInfo.getAlertData();
|
||||
Map<String, String> alertParams = alertInfo.getAlertParams();
|
||||
if (alertParams == null || alertParams.size() == 0) {
|
||||
return new AlertResult("false", "Slack alert params is empty");
|
||||
if (alertParams == null || alertParams.isEmpty()) {
|
||||
return new AlertResult(false, "Slack alert params is empty");
|
||||
}
|
||||
SlackSender slackSender = new SlackSender(alertParams);
|
||||
String response = slackSender.sendMessage(alertData.getTitle(), alertData.getContent());
|
||||
return new AlertResult("ok".equals(response) ? "true" : "false", response);
|
||||
return new AlertResult("ok".equals(response), response);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
|||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
|
@ -81,11 +82,11 @@ public final class SlackSender {
|
|||
}
|
||||
|
||||
HttpPost httpPost = new HttpPost(webHookUrl);
|
||||
httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), "UTF-8"));
|
||||
httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), StandardCharsets.UTF_8));
|
||||
CloseableHttpResponse response = httpClient.execute(httpPost);
|
||||
|
||||
HttpEntity entity = response.getEntity();
|
||||
return EntityUtils.toString(entity, "UTF-8");
|
||||
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
log.error("Send message to slack error.", e);
|
||||
return "System Exception";
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ public final class TelegramAlertChannel implements AlertChannel {
|
|||
public AlertResult process(AlertInfo info) {
|
||||
Map<String, String> alertParams = info.getAlertParams();
|
||||
if (alertParams == null || alertParams.isEmpty()) {
|
||||
return new AlertResult("false", "Telegram alert params is empty");
|
||||
return AlertResult.fail("Telegram alert params is empty");
|
||||
}
|
||||
AlertData data = info.getAlertData();
|
||||
return new TelegramSender(alertParams).sendMessage(data);
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public final class TelegramSender {
|
|||
} catch (Exception e) {
|
||||
log.warn("send telegram alert msg exception : {}", e.getMessage());
|
||||
result = new AlertResult();
|
||||
result.setStatus("false");
|
||||
result.setSuccess(false);
|
||||
result.setMessage(String.format("send telegram alert fail. %s", e.getMessage()));
|
||||
}
|
||||
return result;
|
||||
|
|
@ -113,7 +113,7 @@ public final class TelegramSender {
|
|||
|
||||
private AlertResult parseRespToResult(String resp) {
|
||||
AlertResult result = new AlertResult();
|
||||
result.setStatus("false");
|
||||
result.setSuccess(false);
|
||||
if (null == resp || resp.isEmpty()) {
|
||||
result.setMessage("send telegram msg error. telegram server resp is empty");
|
||||
return result;
|
||||
|
|
@ -127,7 +127,7 @@ public final class TelegramSender {
|
|||
result.setMessage(String.format("send telegram alert fail. telegram server error_code: %d, description: %s",
|
||||
response.errorCode, response.description));
|
||||
} else {
|
||||
result.setStatus("true");
|
||||
result.setSuccess(true);
|
||||
result.setMessage("send telegram msg success.");
|
||||
}
|
||||
return result;
|
||||
|
|
@ -153,7 +153,7 @@ public final class TelegramSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "UTF-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ public class TelegramSenderTest {
|
|||
TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, "XXXXXXX");
|
||||
TelegramSender telegramSender = new TelegramSender(telegramConfig);
|
||||
AlertResult result = telegramSender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -65,7 +65,7 @@ public class TelegramSenderTest {
|
|||
TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, "-XXXXXXX");
|
||||
TelegramSender telegramSender = new TelegramSender(telegramConfig);
|
||||
AlertResult result = telegramSender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -75,7 +75,7 @@ public class TelegramSenderTest {
|
|||
alertData.setContent("telegram test content");
|
||||
TelegramSender telegramSender = new TelegramSender(telegramConfig);
|
||||
AlertResult result = telegramSender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ public class TelegramSenderTest {
|
|||
TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_MARKDOWN);
|
||||
TelegramSender telegramSender = new TelegramSender(telegramConfig);
|
||||
AlertResult result = telegramSender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ public class TelegramSenderTest {
|
|||
TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramAlertConstants.PARSE_MODE_HTML);
|
||||
TelegramSender telegramSender = new TelegramSender(telegramConfig);
|
||||
AlertResult result = telegramSender.sendMessage(alertData);
|
||||
Assertions.assertEquals("false", result.getStatus());
|
||||
Assertions.assertFalse(result.isSuccess());
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue