diff --git a/.gitignore b/.gitignore index a2bc772ba4..9011db1479 100644 --- a/.gitignore +++ b/.gitignore @@ -4,43 +4,45 @@ .zip .gz .DS_Store +.target .idea/ +target/ dist/ all-dependencies.txt self-modules.txt third-party-dependencies.txt -**/target/ .settings .nbproject .classpath .project -**/*.iml +*.iml *.ipr *.iws *.tgz .*.swp .vim .tmp -**/node_modules +node_modules npm-debug.log .vscode logs/* +.mvn/ .www t.* +.factorypath +Chart.lock yarn.lock package-lock.json config.gypi test/coverage /docs/zh_CN/介绍 /docs/zh_CN/贡献代码.md -/dolphinscheduler-common/src/main/resources/zookeeper.properties +dolphinscheduler-common/src/main/resources/zookeeper.properties +dolphinscheduler-dao/src/main/resources/dao/data_source.properties dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist/ +dolphinscheduler-ui/dist dolphinscheduler-ui/node -dolphinscheduler-dao/src/main/resources/dao/data_source.properties - -.mvn/wrapper/*.jar - +docker/build/apache-dolphinscheduler* diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index a870dd3cd6..8a837bf8ae 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -15,53 +15,37 @@ # limitations under the License. # -FROM nginx:alpine +FROM openjdk:8-jdk-alpine ARG VERSION ENV TZ Asia/Shanghai ENV LANG C.UTF-8 -ENV DEBIAN_FRONTEND noninteractive +ENV DOCKER true -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. -#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: -#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# 1. install command/library/software +# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: +# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories RUN apk update && \ - apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ - apk add --update procps && \ - openrc boot + apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + apk del tzdata && \ + rm -rf /var/cache/apk/* -#2. install jdk -RUN apk add openjdk8 -ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk -ENV PATH $JAVA_HOME/bin:$PATH - -#3. add dolphinscheduler +# 2. add dolphinscheduler ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ -RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ +RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install pg -RUN apk add postgresql postgresql-contrib -#5. modify nginx -RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ - rm -rf /etc/nginx/conf.d/* -ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d - -#6. add configuration and modify permissions and set soft links +# 3. add configuration and modify permissions and set soft links ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/checkpoint.sh && \ - chmod +x /root/startup-init-conf.sh && \ - chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ - chmod +x /opt/dolphinscheduler/script/*.sh && \ - chmod +x /opt/dolphinscheduler/bin/*.sh && \ - dos2unix /root/checkpoint.sh && \ +RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ @@ -69,13 +53,10 @@ RUN chmod +x /root/checkpoint.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /tmp/xls - -#7. remove apk index cache and disable coredup for sudo -RUN rm -rf /var/cache/apk/* && \ + mkdir -p /tmp/xls && \ echo "Set disable_coredump false" >> /etc/sudo.conf -#8. expose port -EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 +# 4. expose port +EXPOSE 5678 1234 12345 50051 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] diff --git a/docker/build/README.md b/docker/build/README.md index 2607c5831b..8345a676e8 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ You can specify **existing postgres service**. Example: $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ You can specify **existing zookeeper service**. Example: $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* Start a **frontend**, For example: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` - **Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -302,18 +293,6 @@ This environment variable sets enterprise wechat agent id for `alert-server`. Th This environment variable sets enterprise wechat users for `alert-server`. The default value is empty. -**`FRONTEND_API_SERVER_HOST`** - -This environment variable sets api server host for `frontend`. The default value is `127.0.0.1`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - -**`FRONTEND_API_SERVER_PORT`** - -This environment variable sets api server port for `frontend`. The default value is `123451`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - ## Initialization scripts If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`. @@ -322,7 +301,7 @@ For example, to add an environment variable `API_SERVER_PORT` in `/root/start-in ``` export API_SERVER_PORT=5555 -``` +``` and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port: ``` @@ -339,8 +318,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index c2affc0691..27b7465022 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* 启动一个 **frontend**, 如下: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` - **注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -302,18 +293,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`alert-server`的邮件服务企业微信`USERS`，默认值 `空`。 -**`FRONTEND_API_SERVER_HOST`** - -配置`frontend`的连接`api-server`的地址，默认值 `127.0.0.1`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - -**`FRONTEND_API_SERVER_PORT`** - -配置`frontend`的连接`api-server`的端口，默认值 `12345`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - ## 初始化脚本 如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量，你可以在`/root/start-init-conf.sh`文件中进行修改，同时如果涉及到配置文件的修改，请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件 @@ -322,7 +301,7 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 ``` export API_SERVER_PORT=5555 -``` +``` 当添加以上环境变量后，你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置: ``` @@ -339,8 +318,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/checkpoint.sh b/docker/build/checkpoint.sh old mode 100644 new mode 100755 diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 5d1c07858d..25864fde20 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - ${log.base}/dolphinscheduler-alert.log @@ -45,7 +37,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml index 2df90d8392..6d29f8af5f 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -55,7 +47,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml index 7410c01f05..d1bfb67aa1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -74,7 +66,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml index be1d0acde5..b7e08dd846 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - diff --git a/docker/build/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf deleted file mode 100644 index 9c2c3913dc..0000000000 --- a/docker/build/conf/nginx/dolphinscheduler.conf +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -server { - listen 8888; - server_name localhost; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /opt/dolphinscheduler/ui; - index index.html index.html; - } - location /dolphinscheduler { - proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header x_real_ipP $remote_addr; - proxy_set_header remote_addr $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_connect_timeout 300s; - proxy_read_timeout 300s; - proxy_send_timeout 300s; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/docker/build/hooks/build b/docker/build/hooks/build old mode 100644 new mode 100755 index 3276f167c4..a62a53add6 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -47,7 +47,12 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubati mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ # docker build -echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" -sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/ +BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +echo -e "$BUILD_COMMAND\n" +if (docker info 2> /dev/null | grep -i "ERROR"); then + sudo $BUILD_COMMAND +else + $BUILD_COMMAND +fi echo "------ dolphinscheduler end - build -------" diff --git a/docker/build/hooks/push b/docker/build/hooks/push old mode 100644 new mode 100755 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh old mode 100644 new mode 100755 index ac21fe422b..5b1fe06d3b --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -95,12 +95,6 @@ export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""} export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""} export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""} -#============================================================================ -# Frontend -#============================================================================ -export FRONTEND_API_SERVER_HOST=${FRONTEND_API_SERVER_HOST:-"127.0.0.1"} -export FRONTEND_API_SERVER_PORT=${FRONTEND_API_SERVER_PORT:-"12345"} - echo "generate app config" ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do eval "cat << EOF @@ -108,7 +102,3 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf \ No newline at end of file diff --git a/docker/build/startup.sh b/docker/build/startup.sh old mode 100644 new mode 100755 index 0511788d48..6d0689f88e --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -43,7 +43,7 @@ initDatabase() { exit 1 fi else - v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") + v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then echo "Error: Can't connect to database...${v}" exit 1 @@ -70,12 +70,6 @@ initZK() { done } -# start nginx -initNginx() { - echo "start nginx" - nginx & -} - # start master-server initMasterServer() { echo "start master-server" @@ -115,19 +109,17 @@ initAlertServer() { printUsage() { echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system," echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n" - echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | frontend ]\n" - printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server, alert-server and frontend." + echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server ]\n" + printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server" printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring." - printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services.." - printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests from the front-end UI layer." + printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services." + printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer." printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms." - printf "%-13s: %s\n" "frontend" "Frontend mainly provides various visual operation interfaces of the system." } # init config file source /root/startup-init-conf.sh -LOGFILE=/var/log/nginx/access.log case "$1" in (all) initZK @@ -137,8 +129,7 @@ case "$1" in initApiServer initAlertServer initLoggerServer - initNginx - LOGFILE=/var/log/nginx/access.log + LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (master-server) initZK @@ -164,10 +155,6 @@ case "$1" in initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; - (frontend) - initNginx - LOGFILE=/var/log/nginx/access.log - ;; (help) printUsage exit 1 @@ -183,4 +170,3 @@ mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null > echo "tail begin" exec bash -c "tail -n 1 -f ${LOGFILE}" - diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check old mode 100644 new mode 100755 index 59203c5b3e..f4093664da --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -25,7 +25,7 @@ else echo "Server start failed "$server_num exit 1 fi -ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` +ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` if [ $ready -eq 1 ] then echo "Servers is ready" diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 5ef116edba..a96fb14e22 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -81,29 +81,6 @@ services: networks: - dolphinscheduler - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-frontend - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 30s - depends_on: - - dolphinscheduler-api - volumes: - - dolphinscheduler-logs:/var/log/nginx - networks: - - dolphinscheduler - dolphinscheduler-alert: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-alert @@ -209,7 +186,7 @@ services: timeout: 5s retries: 3 start_period: 30s - depends_on: + depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 06dabce6c3..06cb781121 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -27,6 +27,7 @@ services: POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler + POSTGRESQL_REPLICATION_PASSWORD: dolphinscheduler@123 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -64,37 +65,15 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + ZOOKEEPER_ROOT: /dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - deploy: - mode: replicated - replicas: 1 - - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30 - timeout: 5s - retries: 3 - start_period: 30s - volumes: - - dolphinscheduler-logs:/var/log/nginx networks: - dolphinscheduler deploy: @@ -127,13 +106,13 @@ services: DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: + networks: - dolphinscheduler deploy: mode: replicated @@ -142,9 +121,9 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler:latest command: ["master-server"] - ports: + ports: - 5678:5678 - environment: + environment: TZ: Asia/Shanghai MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" @@ -161,7 +140,7 @@ services: ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s @@ -176,10 +155,10 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler:latest command: ["worker-server"] - ports: + ports: - 1234:1234 - 50051:50051 - environment: + environment: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" @@ -196,7 +175,7 @@ services: ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh old mode 100644 new mode 100755 diff --git a/dolphinscheduler-ui/install-dolphinscheduler-ui.sh b/dolphinscheduler-ui/install-dolphinscheduler-ui.sh old mode 100644 new mode 100755 diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties index 352d9375be..135af0baa0 100644 --- a/e2e/src/test/resources/config/config.properties +++ b/e2e/src/test/resources/config/config.properties @@ -17,7 +17,7 @@ ############### project ############## # login url -LOGIN_URL=http://127.0.0.1:8888/dolphinscheduler/ +LOGIN_URL=http://127.0.0.1:12345/dolphinscheduler/ #login username USER_NAME=admin #login password diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh old mode 100644 new mode 100755 diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh old mode 100644 new mode 100755 index 09f3fe838b..fc10d9f266 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -55,6 +55,12 @@ fi log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid +# print logs to /dev/null in docker +if [ "$DOCKER" = "true" ]; then + echo "start in docker" + log=/dev/null +fi + cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 diff --git a/script/monitor-server.sh b/script/monitor-server.sh old mode 100644 new mode 100755 diff --git a/script/remove-zk-node.sh b/script/remove-zk-node.sh old mode 100644 new mode 100755 diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh old mode 100644 new mode 100755 diff --git a/script/start-all.sh b/script/start-all.sh old mode 100644 new mode 100755 diff --git a/script/stop-all.sh b/script/stop-all.sh old mode 100644 new mode 100755 diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh old mode 100644 new mode 100755