natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split the components into individual package, refactor configurations to use Spring properties (#7160) 

* Split the components into individual package

A follow-up PR will be made to build dedicated Docker images for each
component, so that every component Docker image has minimal jars, which
is easy to maintain and good for security fixes.

* Split the components into individual package

A follow-up PR will be made to build dedicated Docker images for each
component, so that every component Docker image has minimal jars, which
is easy to maintain and good for security fixes.

* Split the components into individual package

A follow-up PR will be made to build dedicated Docker images for each
component, so that every component Docker image has minimal jars, which
is easy to maintain and good for security fixes.
This commit is contained in:
kezhenxu94 2021-12-10 11:28:53 +08:00 committed by GitHub
parent 8d68cf48dd
commit 67cc260d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
282 changed files with 2990 additions and 3379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/backend.yml vendored
View File

@ -47,8 +47,6 @@ jobs:
submodules: true
- name: Sanity Check
uses: ./.github/actions/sanity-check
with:
token: ${{ secrets.GITHUB_TOKEN }} # We only need to pass this token in one workflow
- uses: actions/cache@v2
with:
path: ~/.m2/repository
@ -56,7 +54,7 @@ jobs:
- name: Build and Package
run: |
./mvnw -B clean install \
-Prelease \
-Prelease,docker \
-Dmaven.test.skip=true \
-Dcheckstyle.skip=true \
-Dhttp.keepAlive=false \

8
.github/workflows/e2e.yml vendored
View File

@ -61,7 +61,13 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-
- name: Build Image
run: TAG=ci sh ./docker/build/hooks/build
run: |
./mvnw -B clean install \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dmaven.checkstyle.skip \
-Pdocker,release -Ddocker.tag=ci \
-pl dolphinscheduler-standalone-server -am
- name: Run Test
run: |
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \

1
.gitignore vendored
View File

@ -45,7 +45,6 @@ dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist
dolphinscheduler-ui/node
docker/build/apache-dolphinscheduler*
dolphinscheduler-common/sql
dolphinscheduler-common/test

0
docker/README.md → deploy/README.md
View File

23
dolphinscheduler-dao/src/main/resources/application-postgresql.yaml → deploy/docker/.env Normal file → Executable file
View File

@ -15,20 +15,9 @@
# specific language governing permissions and limitations
# under the License.
#
spring:
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
minimum-idle: 5
auto-commit: true
validation-timeout: 3000
pool-name: DolphinScheduler
maximum-pool-size: 50
connection-timeout: 30000
idle-timeout: 600000
leak-detection-threshold: 0
initialization-fail-timeout: 1
HUB=apache
TAG=latest
TZ=Asia/Shanghai
SPRING_DATASOURCE_URL=jdbc:postgresql://dolphinscheduler-postgresql:5432/dolphinscheduler
REGISTRY_ZOOKEEPER_CONNECT_STRING=dolphinscheduler-zookeeper:2181

153
deploy/docker/docker-compose.yml Normal file
View File

@ -0,0 +1,153 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "3.8"
services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:11.11.0
environment:
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"]
interval: 5s
timeout: 60s
retries: 120
networks:
- dolphinscheduler
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:3.6.2
environment:
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"]
interval: 5s
timeout: 60s
retries: 120
networks:
- dolphinscheduler
dolphinscheduler-schema-initializer:
image: ${HUB}/dolphinscheduler-tools:${TAG}
env_file: .env
command: [ bin/create-schema.sh ]
depends_on:
dolphinscheduler-postgresql:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
dolphinscheduler-api:
image: ${HUB}/dolphinscheduler-api:${TAG}
ports:
- "12345:12345"
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-schema-initializer:
condition: service_completed_successfully
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
dolphinscheduler-alert:
image: ${HUB}/dolphinscheduler-alert-server:${TAG}
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-schema-initializer:
condition: service_completed_successfully
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-master:
image: ${HUB}/dolphinscheduler-master:${TAG}
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-schema-initializer:
condition: service_completed_successfully
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
networks:
- dolphinscheduler
dolphinscheduler-worker:
image: ${HUB}/dolphinscheduler-worker:${TAG}
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-schema-initializer:
condition: service_completed_successfully
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
networks:
dolphinscheduler:
driver: bridge
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local:

77
docker/docker-swarm/docker-stack.yml → deploy/docker/docker-stack.yml
View File

@ -17,7 +17,6 @@
version: "3.1"
services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:11.11.0
environment:
@ -26,9 +25,9 @@ services:
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
@ -40,93 +39,81 @@ services:
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-api:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: api-server
image: apache/dolphinscheduler-tools
ports:
- 12345:12345
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
- 12345:12345
env_file: .env
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-alert:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: alert-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
image: apache/dolphinscheduler-alert-server
env_file: .env
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-master:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: master-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
image: apache/dolphinscheduler-master
env_file: .env
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-worker:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: worker-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
image: apache/dolphinscheduler-worker
env_file: .env
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
@ -141,4 +128,4 @@ volumes:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local:
dolphinscheduler-resource-local:

0
docker/kubernetes/dolphinscheduler/Chart.yaml → deploy/kubernetes/dolphinscheduler/Chart.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/NOTES.txt → deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
View File

0
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl → deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
View File

0
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml → deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml → deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml → deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml → deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml → deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml → deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml → deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/ingress.yaml → deploy/kubernetes/dolphinscheduler/templates/ingress.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml → deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml → deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml → deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml → deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml → deploy/kubernetes/dolphinscheduler/templates/secret-external-database.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml → deploy/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml → deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml → deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml → deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml → deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml → deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
View File

0
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml → deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
View File

0
docker/kubernetes/dolphinscheduler/values.yaml → deploy/kubernetes/dolphinscheduler/values.yaml
View File

71
docker/build/Dockerfile
View File

@ -1,71 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM openjdk:8-jre-slim-buster
ARG VERSION
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ Asia/Shanghai
ENV LANG C.UTF-8
ENV DOCKER true
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
# 1. install command/library/software
# If install slowly, you can replcae debian's mirror with new mirror, Example:
# RUN { \
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster main contrib non-free"; \
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-updates main contrib non-free"; \
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-backports main contrib non-free"; \
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security buster/updates main contrib non-free"; \
# } > /etc/apt/sources.list
RUN apt-get update && \
apt-get install -y --no-install-recommends tzdata dos2unix python supervisor procps psmisc netcat sudo tini && \
echo "Asia/Shanghai" > /etc/timezone && \
rm -f /etc/localtime && \
dpkg-reconfigure tzdata && \
rm -rf /var/lib/apt/lists/* /tmp/*
# 2. add dolphinscheduler
ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/
RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin
# 3. add configuration and modify permissions and set soft links
COPY ./checkpoint.sh /root/checkpoint.sh
COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor/conf.d/
COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
rm -f /bin/sh && \
ln -s /bin/bash /bin/sh && \
mkdir -p /tmp/xls && \
echo PS1=\'\\w \\$ \' >> ~/.bashrc && \
echo "Set disable_coredump false" >> /etc/sudo.conf
# 4. expose port
EXPOSE 5678 1234 12345 50051 50052
ENTRYPOINT ["/usr/bin/tini", "--", "/root/startup.sh"]

91
docker/build/conf/dolphinscheduler/common.properties.tpl
View File

@ -1,91 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=${DATA_BASEDIR_PATH}
# resource storage type: HDFS, S3, NONE
resource.storage.type=${RESOURCE_STORAGE_TYPE}
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.upload.path=${RESOURCE_UPLOAD_PATH}
# whether to startup kerberos
hadoop.security.authentication.startup.state=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE}
# java.security.krb5.conf path
java.security.krb5.conf.path=${JAVA_SECURITY_KRB5_CONF_PATH}
# login user from keytab username
login.user.keytab.username=${LOGIN_USER_KEYTAB_USERNAME}
# login user from keytab path
login.user.keytab.path=${LOGIN_USER_KEYTAB_PATH}
# kerberos expire time, the unit is hour
kerberos.expire.time=${KERBEROS_EXPIRE_TIME}
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
hdfs.root.user=${HDFS_ROOT_USER}
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=${FS_DEFAULT_FS}
# if resource.storage.type=S3, s3 endpoint
fs.s3a.endpoint=${FS_S3A_ENDPOINT}
# if resource.storage.type=S3, s3 access key
fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3, s3 secret key
fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# resourcemanager port, the default value is 8088 if not specified
resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT}
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS}
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.application.status.address=${YARN_APPLICATION_STATUS_ADDRESS}
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
yarn.job.history.status.address=${YARN_JOB_HISTORY_STATUS_ADDRESS}
# datasource encryption enable
datasource.encryption.enable=${DATASOURCE_ENCRYPTION_ENABLE}
# datasource encryption salt
datasource.encryption.salt=${DATASOURCE_ENCRYPTION_SALT}
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=${SUDO_ENABLE}
# network interface preferred like eth0, default: empty
#dolphin.scheduler.network.interface.preferred=
# network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default
# system env path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
# development state
development.state=false

62
docker/build/conf/dolphinscheduler/logback/logback-api.xml
View File

@ -1,62 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- api server logback config start -->
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-api.log</file>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>64MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- api server logback config end -->
<logger name="org.apache.zookeeper" level="WARN"/>
<logger name="org.apache.hbase" level="WARN"/>
<logger name="org.apache.hadoop" level="WARN"/>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/>
</root>
</configuration>

82
docker/build/conf/dolphinscheduler/logback/logback-master.xml
View File

@ -1,82 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter> -->
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
</Discriminator>
<sift>
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender">
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
<append>true</append>
</appender>
</sift>
</appender>
<!-- master server logback config start -->
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-master.log</file>
<!--<filter class="org.apache.dolphinscheduler.server.log.MasterLogFilter">
<level>INFO</level>
</filter>-->
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- master server logback config end -->
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="MASTERLOGFILE"/>
</root>
</configuration>

82
docker/build/conf/dolphinscheduler/logback/logback-worker.xml
View File

@ -1,82 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config start -->
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter> -->
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
</Discriminator>
<sift>
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender">
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
<append>true</append>
</appender>
</sift>
</appender>
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-worker.log</file>
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter> -->
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config end -->
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/>
</root>
</configuration>

19
docker/build/conf/dolphinscheduler/registry.properties.tpl
View File

@ -1,19 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
registry.plugin.name=${REGISTRY_PLUGIN_NAME}
registry.servers=${REGISTRY_SERVERS}

107
docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
View File

@ -1,107 +0,0 @@
; Licensed to the Apache Software Foundation (ASF) under one
; or more contributor license agreements. See the NOTICE file
; distributed with this work for additional information
; regarding copyright ownership. The ASF licenses this file
; to you under the Apache License, Version 2.0 (the
; "License"); you may not use this file except in compliance
; with the License. You may obtain a copy of the License at
;
; http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
; distributed under the License is distributed on an "AS IS" BASIS,
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
; See the License for the specific language governing permissions and
; limitations under the License.
; program config file
[program:master]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_MASTER_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:worker]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_WORKER_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:api]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_API_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:alert]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_ALERT_START_ENABLED)s
autorestart=true
startsecs=5
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:logger]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_LOGGER_START_ENABLED)s
autorestart=true
startsecs=5
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:standalone]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start standalone-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_STANDALONE_START_ENABLED)s
autorestart=true
startsecs=5
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0

34
docker/build/hooks/build
View File

@ -1,34 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
ROOT_DIR=$(dirname "$0")/../../..
MVN="$ROOT_DIR"/mvnw
VERSION=$("$MVN" -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
DOCKER_REPO=${DOCKER_REPO:-"apache/dolphinscheduler"}
TAG=${TAG:-"$VERSION"}
echo "Building Docker image as: $DOCKER_REPO:$TAG"
"$MVN" -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
cp "$ROOT_DIR"/dolphinscheduler-dist/target/apache-dolphinscheduler-$VERSION-bin.tar.gz "$ROOT_DIR"/docker/build/
docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$TAG "$ROOT_DIR"/docker/build/

61
docker/build/hooks/build.bat
View File

@ -1,61 +0,0 @@
:: Licensed to the Apache Software Foundation (ASF) under one or more
:: contributor license agreements. See the NOTICE file distributed with
:: this work for additional information regarding copyright ownership.
:: The ASF licenses this file to You under the Apache License, Version 2.0
:: (the "License"); you may not use this file except in compliance with
:: the License. You may obtain a copy of the License at
::
:: http://www.apache.org/licenses/LICENSE-2.0
::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
:: See the License for the specific language governing permissions and
:: limitations under the License.
::
@echo off
echo "------ dolphinscheduler start - build -------"
set
setlocal enableextensions enabledelayedexpansion
if not defined VERSION (
echo "set environment variable [VERSION]"
set first=1
for /f "tokens=3 delims=<>" %%a in ('findstr "<version>[0-9].*</version>" %cd%\pom.xml') do (
if !first! EQU 1 (set VERSION=%%a)
set first=0
)
)
if not defined DOCKER_REPO (
echo "set environment variable [DOCKER_REPO]"
set DOCKER_REPO=dolphinscheduler
)
echo "Version: %VERSION%"
echo "Repo: %DOCKER_REPO%"
echo "Current Directory is %cd%"
:: maven package(Project Directory)
echo "mvn clean package -Prelease -DskipTests=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
call mvn clean package -Prelease -DskipTests=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
if "%errorlevel%"=="1" goto :mvnFailed
:: move dolphinscheduler-bin.tar.gz file to docker/build directory
echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\"
move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\
:: docker build
echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"
docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\
if "%errorlevel%"=="1" goto :dockerBuildFailed
echo "------ dolphinscheduler end - build -------"
:mvnFailed
echo "MAVEN PACKAGE FAILED!"
:dockerBuildFailed
echo "DOCKER BUILD FAILED!"

23
docker/build/hooks/push.bat
View File

@ -1,23 +0,0 @@
:: Licensed to the Apache Software Foundation (ASF) under one or more
:: contributor license agreements. See the NOTICE file distributed with
:: this work for additional information regarding copyright ownership.
:: The ASF licenses this file to You under the Apache License, Version 2.0
:: (the "License"); you may not use this file except in compliance with
:: the License. You may obtain a copy of the License at
::
:: http://www.apache.org/licenses/LICENSE-2.0
::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
:: See the License for the specific language governing permissions and
:: limitations under the License.
::
@echo off
echo "------ push start -------"
set
docker push %DOCKER_REPO%:%VERSION%
echo "------ push end -------"

143
docker/build/startup-init-conf.sh
View File

@ -1,143 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
echo "init env variables"
# Define parameters default value
#============================================================================
# Database
#============================================================================
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
# export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PORT=${DATABASE_PORT:-"5432"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
# transform params to SPRING_DATASOURCE_XXX
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}"
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME}
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD}
# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER}
#============================================================================
# Registry
#============================================================================
export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}
#============================================================================
# Common
#============================================================================
# common opts
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
# common env
export DATA_BASEDIR_PATH=${DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
export HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE:-"false"}
export JAVA_SECURITY_KRB5_CONF_PATH=${JAVA_SECURITY_KRB5_CONF_PATH:-"/opt/krb5.conf"}
export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM"}
export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"}
export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"}
export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"}
export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"}
export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""}
export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:%s/ws/v1/cluster/apps/%s"}
export YARN_JOB_HISTORY_STATUS_ADDRESS=${YARN_JOB_HISTORY_STATUS_ADDRESS:-"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"}
export DATASOURCE_ENCRYPTION_ENABLE=${DATASOURCE_ENCRYPTION_ENABLE:-"false"}
export DATASOURCE_ENCRYPTION_SALT=${DATASOURCE_ENCRYPTION_SALT:-"!@#$%^&*"}
export SUDO_ENABLE=${SUDO_ENABLE:-"true"}
# dolphinscheduler env
export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"}
export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"}
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
#============================================================================
# Master Server
#============================================================================
export MASTER_SERVER_OPTS=${MASTER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"}
export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"}
export MASTER_DISPATCH_TASK_NUM=${MASTER_DISPATCH_TASK_NUM:-"3"}
export MASTER_HOST_SELECTOR=${MASTER_HOST_SELECTOR:-"LowerWeight"}
export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"}
export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"}
export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"}
export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"-1"}
export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.3"}
#============================================================================
# Worker Server
#============================================================================
export WORKER_SERVER_OPTS=${WORKER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"}
export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"}
export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"}
export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"-1"}
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.3"}
export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"localhost"}
#============================================================================
# Alert Server
#============================================================================
export ALERT_SERVER_OPTS=${ALERT_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
#============================================================================
# Api Server
#============================================================================
export API_SERVER_OPTS=${API_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
#============================================================================
# Logger Server
#============================================================================
export LOGGER_SERVER_OPTS=${LOGGER_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
echo "generate dolphinscheduler config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
# generate dolphinscheduler env
DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
if [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
EOF
" > ${DOLPHINSCHEDULER_ENV_PATH}
chmod +x ${DOLPHINSCHEDULER_ENV_PATH}
fi

132
docker/build/startup.sh
View File

@ -1,132 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
export MASTER_START_ENABLED=false
export WORKER_START_ENABLED=false
export API_START_ENABLED=false
export ALERT_START_ENABLED=false
export LOGGER_START_ENABLED=false
export STANDALONE_START_ENABLED=false
# wait database
waitDatabase() {
echo "try to connect ${DATABASE_TYPE} ..."
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
local counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't connect to ${DATABASE_TYPE}."
exit 1
fi
echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5
done
echo "${DATABASE_TYPE} connection is ok"
}
# init database
initDatabase() {
echo "import sql data"
${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
}
# wait zk
waitZK() {
echo "try to connect zookeeper ..."
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
local counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't connect to zookeeper."
exit 1
fi
echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
sleep 5
done
done
echo "zookeeper connection is ok"
}
# print usage
printUsage() {
echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n"
echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | standalone-server ]\n"
printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server"
printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring."
printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services."
printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer."
printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms."
printf "%-13s: %s\n" "standalone-server" "Standalone server that uses embedded zookeeper and database, only for testing and demostration."
}
# init config file
source /root/startup-init-conf.sh
case "$1" in
(all)
waitZK
waitDatabase
initDatabase
export MASTER_START_ENABLED=true
export WORKER_START_ENABLED=true
export API_START_ENABLED=true
export ALERT_START_ENABLED=true
export LOGGER_START_ENABLED=true
;;
(master-server)
waitZK
waitDatabase
export MASTER_START_ENABLED=true
;;
(worker-server)
waitZK
waitDatabase
export WORKER_START_ENABLED=true
export LOGGER_START_ENABLED=true
;;
(api-server)
waitZK
waitDatabase
initDatabase
export API_START_ENABLED=true
;;
(alert-server)
waitDatabase
export ALERT_START_ENABLED=true
;;
(standalone-server)
export STANDALONE_START_ENABLED=true
;;
(help)
printUsage
exit 1
;;
(*)
printUsage
exit 1
;;
esac
# init directories
mkdir -p ${DOLPHINSCHEDULER_HOME}/logs
# start supervisord
supervisord -n -u root

42
docker/docker-swarm/check
View File

@ -1,42 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
echo "------- check server status -------"
cd $(dirname $0)
until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do
counter=$((counter+1))
if [ $counter -eq 30 ]; then
echo "error: all servers start failed"
exit 1
fi
echo "waiting for all servers started ..."
sleep 5
done
echo "all servers start successfully"
counter=0
until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do
counter=$((counter+1))
if [ $counter -eq 30 ]; then
echo "error: api server is not ready"
exit 1
fi
echo "waiting for api server ready ..."
sleep 5
done
echo "api server is ready"

121
docker/docker-swarm/config.env.sh
View File

@ -1,121 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#============================================================================
# Database
#============================================================================
# postgresql
DATABASE_TYPE=postgresql
DATABASE_DRIVER=org.postgresql.Driver
DATABASE_HOST=dolphinscheduler-postgresql
DATABASE_PORT=5432
DATABASE_USERNAME=root
DATABASE_PASSWORD=root
DATABASE_DATABASE=dolphinscheduler
DATABASE_PARAMS=characterEncoding=utf8
# mysql
# DATABASE_TYPE=mysql
# DATABASE_DRIVER=com.mysql.cj.jdbc.Driver
# DATABASE_HOST=dolphinscheduler-mysql
# DATABASE_PORT=3306
# DATABASE_USERNAME=root
# DATABASE_PASSWORD=root
# DATABASE_DATABASE=dolphinscheduler
# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
#============================================================================
# Registry
#============================================================================
REGISTRY_PLUGIN_NAME=zookeeper
REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181
#============================================================================
# Common
#============================================================================
# common opts
DOLPHINSCHEDULER_OPTS=
# common env
DATA_BASEDIR_PATH=/tmp/dolphinscheduler
RESOURCE_STORAGE_TYPE=HDFS
RESOURCE_UPLOAD_PATH=/dolphinscheduler
FS_DEFAULT_FS=file:///
FS_S3A_ENDPOINT=s3.xxx.amazonaws.com
FS_S3A_ACCESS_KEY=xxxxxxx
FS_S3A_SECRET_KEY=xxxxxxx
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false
JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf
LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM
LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab
KERBEROS_EXPIRE_TIME=2
HDFS_ROOT_USER=hdfs
RESOURCE_MANAGER_HTTPADDRESS_PORT=8088
YARN_RESOURCEMANAGER_HA_RM_IDS=
YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s
YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
DATASOURCE_ENCRYPTION_ENABLE=false
DATASOURCE_ENCRYPTION_SALT=!@#$%^&*
SUDO_ENABLE=true
# dolphinscheduler env
HADOOP_HOME=/opt/soft/hadoop
HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
SPARK_HOME1=/opt/soft/spark1
SPARK_HOME2=/opt/soft/spark2
PYTHON_HOME=/usr/bin/python
JAVA_HOME=/usr/local/openjdk-8
HIVE_HOME=/opt/soft/hive
FLINK_HOME=/opt/soft/flink
DATAX_HOME=/opt/soft/datax
#============================================================================
# Master Server
#============================================================================
MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
MASTER_EXEC_THREADS=100
MASTER_EXEC_TASK_NUM=20
MASTER_DISPATCH_TASK_NUM=3
MASTER_HOST_SELECTOR=LowerWeight
MASTER_HEARTBEAT_INTERVAL=10
MASTER_TASK_COMMIT_RETRYTIMES=5
MASTER_TASK_COMMIT_INTERVAL=1000
MASTER_MAX_CPULOAD_AVG=-1
MASTER_RESERVED_MEMORY=0.3
#============================================================================
# Worker Server
#============================================================================
WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
WORKER_EXEC_THREADS=100
WORKER_HEARTBEAT_INTERVAL=10
WORKER_HOST_WEIGHT=100
WORKER_MAX_CPULOAD_AVG=-1
WORKER_RESERVED_MEMORY=0.3
WORKER_GROUPS=default
ALERT_LISTEN_HOST=dolphinscheduler-alert
#============================================================================
# Alert Server
#============================================================================
ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
#============================================================================
# Api Server
#============================================================================
API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
#============================================================================
# Logger Server
#============================================================================
LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m

143
docker/docker-swarm/docker-compose.yml
View File

@ -1,143 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "3.1"
services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:11.11.0
environment:
TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:3.6.2
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-api:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: api-server
ports:
- 12345:12345
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s
timeout: 5s
retries: 3
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-alert:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: alert-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s
timeout: 5s
retries: 3
depends_on:
- dolphinscheduler-postgresql
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-master:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: master-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
interval: 30s
timeout: 5s
retries: 3
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-worker:
image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
command: worker-server
environment:
TZ: Asia/Shanghai
env_file: config.env.sh
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30s
timeout: 5s
retries: 3
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
restart: unless-stopped
networks:
- dolphinscheduler
networks:
dolphinscheduler:
driver: bridge
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local:

45
dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
View File

@ -85,18 +85,17 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<exclusions>
<exclusion>
<artifactId>jsr305</artifactId>
<groupId>com.google.code.findbugs</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
</dependency>
<!--excel poi-->
<dependency>
<groupId>org.apache.poi</groupId>
@ -124,11 +123,43 @@
<configuration>
<excludes>
<exclude>*.yaml</exclude>
<exclude>*.yml</exclude>
<exclude>*.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-alert-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<finalName>alert-server</finalName>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-alert-server.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>docker</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

64
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml Normal file
View File

@ -0,0 +1,64 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>dolphinscheduler-alert-server</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<baseDirectory>alert-server</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>*.yaml</include>
<include>*.xml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../../script/env</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>dolphinscheduler_env.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>libs</outputDirectory>
</dependencySet>
</dependencySets>
</assembly>

32
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
source "$BIN_DIR/dolphinscheduler_env.sh"
JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
java $JAVA_OPTS \
-cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
org.apache.dolphinscheduler.alert.AlertServer

34
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM openjdk:8-jre-slim-buster
ENV DOCKER true
ENV TZ Asia/Shanghai
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
RUN apt update ; \
apt install -y curl ; \
rm -rf /var/lib/apt/lists/*
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/alert-server $DOLPHINSCHEDULER_HOME
EXPOSE 50052 50053
CMD [ "/bin/bash", "./bin/start.sh" ]

30
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/Monitor.java → dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertConfig.java
View File

@ -14,20 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.monitor;
/**
* server monitor and auto restart server
*/
public interface Monitor {
package org.apache.dolphinscheduler.alert;
/**
* monitor server and restart
*
* @param masterPath masterPath
* @param workerPath workerPath
* @param port port
* @param installPath installPath
*/
void monitor(String masterPath, String workerPath, Integer port, String installPath);
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties("alert")
public final class AlertConfig {
private int port;
public int getPort() {
return port;
}
public void setPort(final int port) {
this.port = port;
}
}

18
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
View File

@ -17,10 +17,7 @@
package org.apache.dolphinscheduler.alert;
import static org.apache.dolphinscheduler.common.Constants.ALERT_RPC_PORT;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
@ -38,15 +35,13 @@ import javax.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@EnableAutoConfiguration
@ComponentScan(value = {
"org.apache.dolphinscheduler.alert",
"org.apache.dolphinscheduler.dao"
})
@SpringBootApplication
@ComponentScan("org.apache.dolphinscheduler")
public class AlertServer implements Closeable {
private static final Logger logger = LoggerFactory.getLogger(AlertServer.class);
@ -58,6 +53,9 @@ public class AlertServer implements Closeable {
private NettyRemotingServer server;
@Autowired
private AlertConfig config;
public AlertServer(PluginDao pluginDao, AlertDao alertDao, AlertPluginManager alertPluginManager, AlertSender alertSender, AlertRequestProcessor alertRequestProcessor) {
this.pluginDao = pluginDao;
this.alertDao = alertDao;
@ -101,7 +99,7 @@ public class AlertServer implements Closeable {
private void startServer() {
NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(PropertyUtils.getInt(ALERT_RPC_PORT, 50052));
serverConfig.setListenPort(config.getPort());
server = new NettyRemotingServer(serverConfig);
server.registerProcessor(CommandType.ALERT_SEND_REQUEST, alertRequestProcessor);

57
dolphinscheduler-api/src/main/resources/application-api.yaml → dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
View File

@ -15,29 +15,30 @@
# limitations under the License.
#
server:
port: 12345
servlet:
session:
timeout: 120m
context-path: /dolphinscheduler/
compression:
enabled: true
mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
jetty:
max-http-form-post-size: 5000000
spring:
application:
name: api-server
jackson:
time-zone: GMT+8
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
name: alert-server
main:
banner-mode: off
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
minimum-idle: 5
auto-commit: true
validation-timeout: 3000
pool-name: DolphinScheduler
maximum-pool-size: 50
connection-timeout: 30000
idle-timeout: 600000
leak-detection-threshold: 0
initialization-fail-timeout: 1
server:
port: 50053
management:
endpoints:
@ -47,3 +48,17 @@ management:
metrics:
tags:
application: ${spring.application.name}
alert:
port: 50052
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

10
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-alert.xml → dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-spring.xml
View File

@ -16,10 +16,9 @@
~ limitations under the License.
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds">
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
@ -45,8 +44,11 @@
</appender>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<if condition="${DOCKER:-false}">
<then>
<appender-ref ref="STDOUT"/>
</then>
</if>
<appender-ref ref="ALERTLOGFILE"/>
</root>
</configuration>

5
dolphinscheduler-alert/pom.xml
View File

@ -41,10 +41,5 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

49
dolphinscheduler-api/pom.xml
View File

@ -45,6 +45,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-ui</artifactId>
</dependency>
<!--springboot-->
<dependency>
<groupId>org.springframework.boot</groupId>
@ -112,7 +117,7 @@
</exclusion>
<exclusion>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP-java6</artifactId>
<artifactId>HikariCP-java7</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -212,6 +217,11 @@
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
</dependency>
<!-- just for test -->
<dependency>
<groupId>org.springframework.boot</groupId>
@ -239,11 +249,10 @@
<directory>${project.basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>sql/**</include>
<include>*.yaml</include>
</includes>
</testResource>
<testResource>
<directory>${project.basedir}/../dolphinscheduler-service/src/main/resources</directory>
<directory>${project.basedir}/src/test/resources</directory>
</testResource>
</testResources>
<plugins>
@ -253,11 +262,43 @@
<configuration>
<excludes>
<exclude>*.yaml</exclude>
<exclude>*.yml</exclude>
<exclude>*.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-api-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<finalName>api-server</finalName>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-api-server.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>docker</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

68
dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml Normal file
View File

@ -0,0 +1,68 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>dolphinscheduler-api-server</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<baseDirectory>api-server</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>*.yaml</include>
<include>*.xml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../script/env</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>dolphinscheduler_env.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
<outputDirectory>./ui</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>libs</outputDirectory>
</dependencySet>
</dependencySets>
</assembly>

32
dolphinscheduler-api/src/main/bin/start.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
source "$BIN_DIR/dolphinscheduler_env.sh"
JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
java $JAVA_OPTS \
-cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
org.apache.dolphinscheduler.api.ApiApplicationServer

34
dolphinscheduler-api/src/main/docker/Dockerfile Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM openjdk:8-jre-slim-buster
ENV DOCKER true
ENV TZ Asia/Shanghai
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
RUN apt update ; \
apt install -y curl ; \
rm -rf /var/lib/apt/lists/*
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/api-server $DOLPHINSCHEDULER_HOME
EXPOSE 12345
CMD [ "/bin/bash", "./bin/start.sh" ]

19
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
View File

@ -17,25 +17,18 @@
package org.apache.dolphinscheduler.api;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
@EnableAutoConfiguration
@ServletComponentScan
@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
@ComponentScan.Filter(type = FilterType.REGEX, pattern = {
"org.apache.dolphinscheduler.server.*",
"org.apache.dolphinscheduler.alert.*"
})
})
public class ApiApplicationServer extends SpringBootServletInitializer {
@SpringBootApplication
@ComponentScan("org.apache.dolphinscheduler")
public class ApiApplicationServer {
public static void main(String[] args) {
new SpringApplicationBuilder(ApiApplicationServer.class).profiles("api").run(args);
SpringApplication.run(ApiApplicationServer.class);
}
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
View File

@ -127,7 +127,7 @@ public class CacheEvictAspect {
SpelExpressionParser spelParser = new SpelExpressionParser();
EvaluationContext ctx = new StandardEvaluationContext();
for (int i = 0; i < paramNameList.size(); i++) {
ctx.setVariable(paramNameList.get(i), paramList.get(i));
ctx.setVariable("p" + i, paramList.get(i));
}
Object obj = spelParser.parseExpression(key).getValue(ctx);
if (null == obj) {

116
dolphinscheduler-api/src/main/resources/application.yaml Normal file
View File

@ -0,0 +1,116 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
server:
port: 12345
servlet:
session:
timeout: 120m
context-path: /dolphinscheduler/
compression:
enabled: true
mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
jetty:
max-http-form-post-size: 5000000
spring:
application:
name: api-server
main:
banner-mode: off
jackson:
time-zone: GMT+8
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
minimum-idle: 5
auto-commit: true
validation-timeout: 3000
pool-name: DolphinScheduler
maximum-pool-size: 50
connection-timeout: 30000
idle-timeout: 600000
leak-detection-threshold: 0
initialization-fail-timeout: 1
quartz:
auto-startup: false
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.threadPool.makeThreadsDaemons: true
org.quartz.threadPool.threadCount: 25
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
management:
endpoints:
web:
exposure:
include: '*'
metrics:
tags:
application: ${spring.application.name}
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate

14
dolphinscheduler-api/src/main/resources/logback-api.xml → dolphinscheduler-api/src/main/resources/logback-spring.xml
View File

@ -16,10 +16,9 @@
~ limitations under the License.
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<configuration scan="true" scanPeriod="120 seconds">
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
@ -29,7 +28,6 @@
</encoder>
</appender>
<!-- api server logback config start -->
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-api.log</file>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
@ -47,15 +45,17 @@
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- api server logback config end -->
<logger name="org.apache.zookeeper" level="WARN"/>
<logger name="org.apache.hbase" level="WARN"/>
<logger name="org.apache.hadoop" level="WARN"/>
<root level="INFO">
<if condition="${DOCKER:-false}">
<then>
<appender-ref ref="STDOUT"/>
</then>
</if>
<appender-ref ref="APILOGFILE"/>
</root>
</configuration>

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
View File

@ -53,7 +53,6 @@ import org.springframework.test.web.servlet.MockMvc;
/**
* abstract controller test
*/
@ActiveProfiles(value = {ProfileType.H2})
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {ApiApplicationServer.class, DaoConfiguration.class, RegistryServer.class})
@AutoConfigureMockMvc
@ -112,12 +111,11 @@ public abstract class AbstractControllerTest {
}
@Configuration
@Profile(ProfileType.H2)
public static class RegistryServer {
@PostConstruct
public void startEmbedRegistryServer() throws Exception {
final TestingServer server = new TestingServer(true);
System.setProperty("registry.servers", server.getConnectString());
System.setProperty("registry.zookeeper.connect-string", server.getConnectString());
}
}
}

22
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application-alert.yaml → dolphinscheduler-api/src/test/resources/application.yaml
View File

@ -16,17 +16,13 @@
#
spring:
application:
name: alert-server
main:
banner-mode: off
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
username: sa
password: ""
server:
port: 50053
management:
endpoints:
web:
exposure:
include: '*'
metrics:
tags:
application: ${spring.application.name}
registry:
type: zookeeper

38
dolphinscheduler-common/pom.xml
View File

@ -46,13 +46,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>jsr305</artifactId>
<groupId>com.google.code.findbugs</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@ -139,10 +132,6 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</exclusion>
<exclusion>
<groupId>xmlenc</groupId>
<artifactId>xmlenc</artifactId>
@ -159,10 +148,6 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<artifactId>jsr305</artifactId>
<groupId>com.google.code.findbugs</groupId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@ -179,10 +164,6 @@
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>xz</artifactId>
<groupId>org.tukaani</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@ -346,10 +327,6 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@ -362,10 +339,6 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
@ -579,22 +552,11 @@
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-annotation</artifactId>
<version>${mybatis-plus.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
View File

@ -38,11 +38,6 @@ public final class Constants {
*/
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
/**
* alert properties
*/
public static final String ALERT_RPC_PORT = "alert.rpc.port";
/**
* registry properties
*/

21
dolphinscheduler-dao/pom.xml
View File

@ -34,6 +34,21 @@
<artifactId>dolphinscheduler-common</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
@ -103,17 +118,11 @@
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

4
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.dao.datasource;
import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType;
@ -27,7 +26,6 @@ import java.util.Properties;
import javax.sql.DataSource;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
@ -69,7 +67,7 @@ public class SpringConnectionFactory {
GlobalConfig.DbConfig dbConfig = new GlobalConfig.DbConfig();
dbConfig.setIdType(IdType.AUTO);
GlobalConfig globalConfig = new GlobalConfig();
GlobalConfig globalConfig = new GlobalConfig().setBanner(false);
globalConfig.setDbConfig(dbConfig);
sqlSessionFactoryBean.setGlobalConfig(globalConfig);
sqlSessionFactoryBean.setTypeAliasesPackage("org.apache.dolphinscheduler.dao.entity");

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
View File

@ -53,7 +53,7 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
/**
* update
*/
@CacheEvict(key = "#processDefinition.code")
@CacheEvict(key = "#p0.code")
int updateById(@Name("processDefinition") @Param("et") ProcessDefinition processDefinition);
/**

34
dolphinscheduler-dao/src/main/resources/application-h2.yaml
View File

@ -1,34 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
spring:
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
username: sa
password: ""
hikari:
connection-test-query: select 1
minimum-idle: 5
auto-commit: true
validation-timeout: 3000
pool-name: DolphinScheduler
maximum-pool-size: 50
connection-timeout: 30000
idle-timeout: 600000
leak-detection-threshold: 0
initialization-fail-timeout: 1

34
dolphinscheduler-dao/src/main/resources/application-mysql.yaml
View File

@ -1,34 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
spring:
datasource:
driver-class-name: com.mysql.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
username: ds_user
password: dolphinscheduler
hikari:
connection-test-query: select 1
minimum-idle: 5
auto-commit: true
validation-timeout: 3000
pool-name: DolphinScheduler
maximum-pool-size: 50
connection-timeout: 30000
idle-timeout: 600000
leak-detection-threshold: 0
initialization-fail-timeout: 1

2
dolphinscheduler-dao/src/main/resources/sql/soft_version
View File

@ -1 +1 @@
2.0.0
2.0.1

4
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
View File

@ -17,13 +17,10 @@
package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.common.enums.ProfileType;
import org.junit.runner.RunWith;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.transaction.annotation.Transactional;
@ -31,7 +28,6 @@ import org.springframework.transaction.annotation.Transactional;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = DaoConfiguration.class)
@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class)
@ActiveProfiles(value = ProfileType.H2)
@Transactional
@Rollback
@EnableTransactionManagement

21
dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml → dolphinscheduler-dao/src/test/resources/application.yaml
View File

@ -16,19 +16,8 @@
#
spring:
application:
name: standalone-server
server:
port: 12345
management:
endpoints:
web:
exposure:
include: '*'
server:
port: 8080
metrics:
tags:
application: ${spring.application.name}
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
username: sa
password: ""

8
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml
View File

@ -135,10 +135,6 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@ -151,10 +147,6 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>

19
dolphinscheduler-dist/pom.xml
View File

@ -51,6 +51,11 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert-server</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-ui</artifactId>
</dependency>
</dependencies>
<profiles>
@ -92,20 +97,6 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

11
dolphinscheduler-dist/release-docs/LICENSE
View File

@ -233,7 +233,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0
commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0
commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0
commons-compress 1.19: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.19, Apache 2.0
commons-configuration 1.10: https://mvnrepository.com/artifact/commons-configuration/commons-configuration/1.10, Apache 2.0
commons-daemon 1.0.13 https://mvnrepository.com/artifact/commons-daemon/commons-daemon/1.0.13, Apache 2.0
commons-dbcp 1.4: https://github.com/apache/commons-dbcp, Apache 2.0
@ -321,7 +321,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
jetty-xml 9.4.44.v20210927: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-xml/9.4.44.v20210927, Apache 2.0 and EPL 1.0
jna 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna/4.5.2, Apache 2.0 and LGPL 2.1
jna-platform 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna-platform/4.5.2, Apache 2.0 and LGPL 2.1
joda-time 2.5: https://github.com/JodaOrg/joda-time, Apache 2.0
joda-time 2.10.13: https://github.com/JodaOrg/joda-time, Apache 2.0
jpam 1.1: https://mvnrepository.com/artifact/net.sf.jpam/jpam/1.1, Apache 2.0
jsqlparser 2.1: https://github.com/JSQLParser/JSqlParser, Apache 2.0 or LGPL 2.1
jsr305 3.0.0: https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305, Apache 2.0
@ -348,8 +348,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
poi 4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi/4.1.2, Apache 2.0
poi-ooxml 4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml/4.1.2, Apache 2.0
poi-ooxml-schemas-4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas/4.1.2, Apache 2.0
quartz 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.0, Apache 2.0
quartz-jobs 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.0, Apache 2.0
quartz 2.3.2: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.2, Apache 2.0
quartz-jobs 2.3.2: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.2, Apache 2.0
snakeyaml 1.28: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.28, Apache 2.0
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
@ -426,6 +426,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
xmlenc 0.52: https://mvnrepository.com/artifact/xmlenc/xmlenc/0.52, BSD
py4j 0.10.9: https://mvnrepository.com/artifact/net.sf.py4j/py4j/0.10.9, BSD 2-clause
LatencyUtils 2.0.3: https://github.com/LatencyUtils/LatencyUtils, BSD-2-Clause
janino 3.1.6: https://mvnrepository.com/artifact/org.codehaus.janino/janino/3.1.6, BSD 3-clause
commons-compiler 3.1.6: https://mvnrepository.com/artifact/org.codehaus.janino/janino/3.1.6, BSD 3-clause
========================================================================
CDDL licenses
@ -490,7 +492,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
Public Domain licenses
========================================================================
xz 1.0: https://mvnrepository.com/artifact/org.tukaani/xz/1.0, Public Domain
aopalliance 1.0: https://mvnrepository.com/artifact/aopalliance/aopalliance/1.0, Public Domain
========================================

31
dolphinscheduler-dist/release-docs/licenses/LICENSE-janino.txt Normal file
View File

@ -0,0 +1,31 @@
Janino - An embedded Java[TM] compiler
Copyright (c) 2001-2016, Arno Unkrig
Copyright (c) 2015-2016 TIBCO Software Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials
provided with the distribution.
3. Neither the name of JANINO nor the names of its contributors
may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

166
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
View File

@ -15,11 +15,9 @@
~ limitations under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>bin</id>
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd"> <id>bin</id>
<formats>
<format>tar.gz</format>
</formats>
@ -28,186 +26,58 @@
<fileSets>
<fileSet>
<directory>${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.ftl</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server</directory>
<outputDirectory>alert-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-api/target/api-server</directory>
<outputDirectory>api-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-master/target/master-server</directory>
<outputDirectory>master-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<excludes>
<exclude>org/apache/dolphinscheduler/dao/mapper/*.xml</exclude>
</excludes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-worker/target/worker-server</directory>
<outputDirectory>worker-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-server/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-log-server/target/logger-server</directory>
<outputDirectory>logger-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-service/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server</directory>
<outputDirectory>standalone-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-meter/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-standalone-server/src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
</includes>
<outputDirectory>conf</outputDirectory>
<directory>${basedir}/../dolphinscheduler-tools/target/tools</directory>
<outputDirectory>tools</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dist/target/dolphinscheduler-dist-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>./ui</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources/sql</directory>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>./sql/sql</outputDirectory> <!-- make sure the hierarchy is the same as in src/resources -->
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>*.*</include>
</includes>
<outputDirectory>./script</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>env/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
<include>status-all.sh</include>
</includes>
<outputDirectory>./bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/.././</directory>
<includes>
<include>*.sh</include>
<include>*.py</include>
<include>DISCLAIMER</include>
</includes>
<outputDirectory>.</outputDirectory>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/release-docs</directory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<excludes>
<exclude>org.apache.dolphinscheduler:dolphinscheduler-dist</exclude>
<exclude>javax.servlet:servlet-api</exclude>
<exclude>org.eclipse.jetty.aggregate:jetty-all</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>

2
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java
View File

@ -53,7 +53,7 @@ class ProjectE2ETest {
@Test
@Order(30)
void testDeleteProject() {
final var page = new ProjectPage(browser);
final ProjectPage page = new ProjectPage(browser);
page.delete(project);
await().untilAsserted(() -> {

4
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/TenantE2ETest.java
View File

@ -58,7 +58,7 @@ class TenantE2ETest {
@Test
@Order(20)
void testCreateDuplicateTenant() {
final var page = new TenantPage(browser);
final TenantPage page = new TenantPage(browser);
page.create(tenant);
@ -73,7 +73,7 @@ class TenantE2ETest {
@Test
@Order(30)
void testDeleteTenant() {
final var page = new TenantPage(browser);
final TenantPage page = new TenantPage(browser);
page.delete(tenant);
await().untilAsserted(() -> {

20
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java
View File

@ -19,12 +19,10 @@
package org.apache.dolphinscheduler.e2e.cases;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
import org.apache.dolphinscheduler.e2e.pages.LoginPage;
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectDetailPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm.TaskType;
@ -34,13 +32,15 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.SubWorkflowTaskForm;
import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.openqa.selenium.remote.RemoteWebDriver;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml")
class WorkflowE2ETest {
private static final String project = "test-workflow-1";
@ -81,9 +81,9 @@ class WorkflowE2ETest {
@Test
@Order(1)
void testCreateWorkflow() {
final var workflow = "test-workflow-1";
final String workflow = "test-workflow-1";
final var workflowDefinitionPage =
final WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goTo(project)
.goToTab(WorkflowDefinitionTab.class);
@ -114,9 +114,9 @@ class WorkflowE2ETest {
@Test
@Order(10)
void testCreateSubWorkflow() {
final var workflow = "test-sub-workflow-1";
final String workflow = "test-sub-workflow-1";
final var workflowDefinitionPage =
final WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(project)
@ -145,9 +145,9 @@ class WorkflowE2ETest {
@Test
@Order(30)
void testRunWorkflow() {
final var workflow = "test-workflow-1";
final String workflow = "test-workflow-1";
final var projectPage =
final ProjectDetailPage projectPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(project);

12
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowDefinitionTab.java
View File

@ -19,20 +19,18 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow;
import lombok.Getter;
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectDetailPage;
import java.util.List;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.FindBys;
import lombok.Getter;
import java.util.List;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@Getter
public final class WorkflowDefinitionTab extends NavBarPage implements ProjectDetailPage.Tab {
@ -94,7 +92,7 @@ public final class WorkflowDefinitionTab extends NavBarPage implements ProjectDe
.filter(WebElement::isDisplayed)
.collect(Collectors.toList());
for (var buttons = cancelButtons.get();
for (List<WebElement> buttons = cancelButtons.get();
!buttons.isEmpty();
buttons = cancelButtons.get()) {
buttons.forEach(WebElement::click);

8
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java
View File

@ -55,11 +55,11 @@ public final class WorkflowForm {
@SneakyThrows
@SuppressWarnings("unchecked")
public <T> T addTask(TaskType type) {
final var task = driver.findElement(By.className("task-item-" + type.name()));
final var canvas = driver.findElement(By.className("dag-container"));
final WebElement task = driver.findElement(By.className("task-item-" + type.name()));
final WebElement canvas = driver.findElement(By.className("dag-container"));
final var js = (JavascriptExecutor) driver;
final var dragAndDrop = String.join("\n",
final JavascriptExecutor js = (JavascriptExecutor) driver;
final String dragAndDrop = String.join("\n",
Resources.readLines(Resources.getResource("dragAndDrop.js"), StandardCharsets.UTF_8));
js.executeScript(dragAndDrop, task, canvas);

13
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowSaveDialog.java
View File

@ -19,9 +19,7 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow;
import java.util.List;
import java.util.stream.Stream;
import lombok.Getter;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
@ -32,7 +30,8 @@ import org.openqa.selenium.support.pagefactory.ByChained;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import lombok.Getter;
import java.util.List;
import java.util.stream.Stream;
@Getter
public final class WorkflowSaveDialog {
@ -72,7 +71,7 @@ public final class WorkflowSaveDialog {
public WorkflowSaveDialog tenant(String tenant) {
selectTenant().click();
final var optionsLocator = By.className("option-tenants");
final By optionsLocator = By.className("option-tenants");
new WebDriverWait(driver, 10)
.until(ExpectedConditions.visibilityOfElementLocated(optionsLocator));
@ -91,9 +90,9 @@ public final class WorkflowSaveDialog {
public WorkflowSaveDialog addGlobalParam(String key, String val) {
assert inputParamKey().size() == inputParamVal().size();
final var len = inputParamKey().size();
final int len = inputParamKey().size();
final var driver = parent().driver();
final WebDriver driver = parent().driver();
Stream.concat(
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add"))).stream(),
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add-dp"))).stream())

15
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java
View File

@ -19,19 +19,18 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow.task;
import lombok.Getter;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
import java.util.List;
import java.util.stream.Stream;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.FindBys;
import org.openqa.selenium.support.PageFactory;
import org.openqa.selenium.support.pagefactory.ByChained;
import lombok.Getter;
import java.util.List;
import java.util.stream.Stream;
@Getter
public abstract class TaskNodeForm {
@ -55,7 +54,7 @@ public abstract class TaskNodeForm {
TaskNodeForm(WorkflowForm parent) {
this.parent = parent;
final var driver = parent.driver();
final WebDriver driver = parent.driver();
PageFactory.initElements(driver, this);
}
@ -69,9 +68,9 @@ public abstract class TaskNodeForm {
public TaskNodeForm addParam(String key, String val) {
assert inputParamKey().size() == inputParamVal().size();
final var len = inputParamKey().size();
final int len = inputParamKey().size();
final var driver = parent().driver();
final WebDriver driver = parent().driver();
Stream.concat(
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add"))).stream(),
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add-dp"))).stream())

7
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
View File

@ -19,17 +19,16 @@ version: "2.1"
services:
dolphinscheduler:
image: apache/dolphinscheduler:ci
command: [ standalone-server ]
image: apache/dolphinscheduler-standalone-server:ci
environment:
DATABASE_TYPE: h2
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
networks:
- e2e
healthcheck:
test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/12345" ]
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 60s
retries: 120

4
dolphinscheduler-e2e/pom.xml
View File

@ -31,8 +31,8 @@
</modules>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<junit.version>5.7.2</junit.version>

89
dolphinscheduler-log-server/pom.xml Normal file
View File

@ -0,0 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-log-server</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-service</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-server</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-logger-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<finalName>logger-server</finalName>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-log-server.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>docker</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

56
dolphinscheduler-log-server/src/main/assembly/dolphinscheduler-log-server.xml Normal file
View File

@ -0,0 +1,56 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>dolphinscheduler-logger-server</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<baseDirectory>logger-server</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../script/env</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>dolphinscheduler_env.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>libs</outputDirectory>
</dependencySet>
</dependencySets>
</assembly>

32
dolphinscheduler-log-server/src/main/bin/start.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
source "$BIN_DIR/dolphinscheduler_env.sh"
JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
java $JAVA_OPTS \
-cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
org.apache.dolphinscheduler.server.log.LoggerServer

34
dolphinscheduler-log-server/src/main/docker/Dockerfile Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM openjdk:8-jre-slim-buster
ENV DOCKER true
ENV TZ Asia/Shanghai
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
RUN apt update ; \
apt install -y curl ; \
rm -rf /var/lib/apt/lists/*
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/logger-server $DOLPHINSCHEDULER_HOME
EXPOSE 50051
CMD [ "/bin/bash", "./bin/start.sh" ]

0
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java → dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
View File

0
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java → dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
View File

5
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java → dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
View File

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.junit.Test;
import org.junit.Test.None;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
@ -37,7 +36,7 @@ import io.netty.channel.Channel;
@PrepareForTest({LoggerUtils.class})
public class LoggerRequestProcessorTest {
@Test(expected = None.class)
@Test
public void testProcessViewWholeLogRequest() {
Channel channel = PowerMockito.mock(Channel.class);
PowerMockito.when(channel.writeAndFlush(Mockito.any(Command.class))).thenReturn(null);
@ -53,4 +52,4 @@ public class LoggerRequestProcessorTest {
LoggerRequestProcessor loggerRequestProcessor = new LoggerRequestProcessor();
loggerRequestProcessor.process(channel, command);
}
}
}

0
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java → dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
View File

0
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java → dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
View File

0
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java → dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
View File

0
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java → dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
View File

Some files were not shown because too many files have changed in this diff Show More