[Improvement][Docker/K8s] Support more configurations and service accesses, reduce the duplication configurations, improve image for python and the compatibility of python home (#5393)
* [Improvement][Docker] Support more configurations and improve image for python * [Improvement][K8s] Support more configurations, more service accesses, and reduce the duplication configurations * [Improvement][Python] Improve the compatibility of python home * [Improvement][Install] Fix install config and install.sh * [Improvement][Install] Fix workflow ut * [Improvement][Docker] Optimize docker-swarm/check script * [Improvement][DB] Update default username and password of database * Update comments in master/worker.properties * Specify version for all images * [Improvement][Docker] Optimize PS1 and WORKDIR * [Improvement][SQL] Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh * [Fix-5431][K8s] Fix master and worker cannot get the right address with custom DNS
This commit is contained in:
Shiwen Cheng
GitHub
parent
9ba4ffbe48
commit
b537f4eb26
|
|
@ -47,10 +47,11 @@ jobs:
|
|||
${{ runner.os }}-maven-
|
||||
- name: Bootstrap database
|
||||
run: |
|
||||
sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml
|
||||
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up --no-start --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
|
||||
sudo cp $(pwd)/sql/dolphinscheduler_postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
|
||||
sed -i "/image: bitnami\/postgresql/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml
|
||||
sed -i "/image: bitnami\/zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml
|
||||
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
|
||||
until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done
|
||||
docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql"
|
||||
- name: Set up JDK 1.8
|
||||
uses: actions/setup-java@v1
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -1 +1,11 @@
|
|||
# DolphinScheduler for Docker
|
||||
# DolphinScheduler for Docker and Kubernetes
|
||||
|
||||
### QuickStart in Docker
|
||||
|
||||
[](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html)
|
||||
[](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/docker-deployment.html)
|
||||
|
||||
### QuickStart in Kubernetes
|
||||
|
||||
[](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html)
|
||||
[](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/kubernetes-deployment.html)
|
||||
|
|
|
|||
|
|
@ -15,28 +15,35 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM openjdk:8-jdk-alpine
|
||||
FROM openjdk:8-jre-slim-buster
|
||||
|
||||
ARG VERSION
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
ENV TZ Asia/Shanghai
|
||||
ENV LANG C.UTF-8
|
||||
ENV DOCKER true
|
||||
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
|
||||
|
||||
# 1. install command/library/software
|
||||
# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
|
||||
# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
|
||||
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
|
||||
RUN apk update && \
|
||||
apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \
|
||||
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
|
||||
apk del tzdata && \
|
||||
rm -rf /var/cache/apk/*
|
||||
# If install slowly, you can replcae debian's mirror with new mirror, Example:
|
||||
# RUN { \
|
||||
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster main contrib non-free"; \
|
||||
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-updates main contrib non-free"; \
|
||||
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-backports main contrib non-free"; \
|
||||
# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security buster/updates main contrib non-free"; \
|
||||
# } > /etc/apt/sources.list
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends tzdata dos2unix python supervisor procps psmisc netcat sudo tini && \
|
||||
echo "Asia/Shanghai" > /etc/timezone && \
|
||||
rm -f /etc/localtime && \
|
||||
dpkg-reconfigure tzdata && \
|
||||
rm -rf /var/lib/apt/lists/* /tmp/*
|
||||
|
||||
# 2. add dolphinscheduler
|
||||
ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/
|
||||
RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
|
||||
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
|
||||
RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
|
||||
WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin
|
||||
|
||||
# 3. add configuration and modify permissions and set soft links
|
||||
COPY ./checkpoint.sh /root/checkpoint.sh
|
||||
|
|
@ -44,19 +51,21 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh
|
|||
COPY ./startup.sh /root/startup.sh
|
||||
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
|
||||
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
|
||||
COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/
|
||||
COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor/conf.d/
|
||||
COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
|
||||
RUN dos2unix /root/checkpoint.sh && \
|
||||
RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \
|
||||
dos2unix /root/checkpoint.sh && \
|
||||
dos2unix /root/startup-init-conf.sh && \
|
||||
dos2unix /root/startup.sh && \
|
||||
dos2unix /opt/dolphinscheduler/script/*.sh && \
|
||||
dos2unix /opt/dolphinscheduler/bin/*.sh && \
|
||||
rm -rf /bin/sh && \
|
||||
rm -f /bin/sh && \
|
||||
ln -s /bin/bash /bin/sh && \
|
||||
mkdir -p /var/mail /tmp/xls && \
|
||||
mkdir -p /tmp/xls && \
|
||||
echo PS1=\'\\w \\$ \' >> ~/.bashrc && \
|
||||
echo "Set disable_coredump false" >> /etc/sudo.conf
|
||||
|
||||
# 4. expose port
|
||||
EXPOSE 5678 1234 12345 50051 50052
|
||||
|
||||
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/root/startup.sh"]
|
||||
|
|
|
|||
|
|
@ -1,488 +0,0 @@
|
|||
## What is DolphinScheduler?
|
||||
|
||||
DolphinScheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.
|
||||
|
||||
GitHub URL: https://github.com/apache/dolphinscheduler
|
||||
|
||||
Official Website: https://dolphinscheduler.apache.org
|
||||
|
||||
|
||||
|
||||
[](README.md)
|
||||
[](README_zh_CN.md)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Docker](https://docs.docker.com/engine/) 1.13.1+
|
||||
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
|
||||
|
||||
## How to use this docker image
|
||||
|
||||
#### You can start a dolphinscheduler by docker-compose (recommended)
|
||||
|
||||
```
|
||||
$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
The default **postgres** user `root`, postgres password `root` and database `dolphinscheduler` are created in the `docker-compose.yml`.
|
||||
|
||||
The default **zookeeper** is created in the `docker-compose.yml`.
|
||||
|
||||
Access the Web UI: http://192.168.xx.xx:12345/dolphinscheduler
|
||||
|
||||
The default username is `admin` and the default password is `dolphinscheduler123`
|
||||
|
||||
> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds`
|
||||
|
||||
#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
|
||||
|
||||
You can specify **existing postgres and zookeeper service**. Example:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-p 12345:12345 \
|
||||
apache/dolphinscheduler:latest all
|
||||
```
|
||||
|
||||
Access the Web UI：http://192.168.xx.xx:12345/dolphinscheduler
|
||||
|
||||
#### Or start a standalone dolphinscheduler server
|
||||
|
||||
You can start a standalone dolphinscheduler server.
|
||||
|
||||
* Create a **local volume** for resource storage, For example:
|
||||
|
||||
```
|
||||
docker volume create dolphinscheduler-resource-local
|
||||
```
|
||||
|
||||
* Start a **master server**, For example:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-master \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
apache/dolphinscheduler:latest master-server
|
||||
```
|
||||
|
||||
* Start a **worker server** (including **logger server**), For example:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-worker \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-e ALERT_LISTEN_HOST="dolphinscheduler-alert" \
|
||||
-v dolphinscheduler-resource-local:/dolphinscheduler \
|
||||
apache/dolphinscheduler:latest worker-server
|
||||
```
|
||||
|
||||
* Start a **api server**, For example:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-api \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-v dolphinscheduler-resource-local:/dolphinscheduler \
|
||||
-p 12345:12345 \
|
||||
apache/dolphinscheduler:latest api-server
|
||||
```
|
||||
|
||||
* Start a **alert server**, For example:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-alert \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
apache/dolphinscheduler:latest alert-server
|
||||
```
|
||||
|
||||
**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
|
||||
|
||||
## How to build a docker image
|
||||
|
||||
You can build a docker image in A Unix-like operating system, You can also build it in Windows operating system.
|
||||
|
||||
In Unix-Like, Example:
|
||||
|
||||
```bash
|
||||
$ cd path/dolphinscheduler
|
||||
$ sh ./docker/build/hooks/build
|
||||
```
|
||||
|
||||
In Windows, Example:
|
||||
|
||||
```bat
|
||||
C:\dolphinscheduler>.\docker\build\hooks\build.bat
|
||||
```
|
||||
|
||||
Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
|
||||
|
||||
## Environment Variables
|
||||
|
||||
The DolphinScheduler Docker container is configured through environment variables, and the default value will be used if an environment variable is not set.
|
||||
|
||||
**`DATABASE_TYPE`**
|
||||
|
||||
This environment variable sets the type for database. The default value is `postgresql`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_DRIVER`**
|
||||
|
||||
This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_HOST`**
|
||||
|
||||
This environment variable sets the host for database. The default value is `127.0.0.1`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_PORT`**
|
||||
|
||||
This environment variable sets the port for database. The default value is `5432`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_USERNAME`**
|
||||
|
||||
This environment variable sets the username for database. The default value is `root`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_PASSWORD`**
|
||||
|
||||
This environment variable sets the password for database. The default value is `root`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_DATABASE`**
|
||||
|
||||
This environment variable sets the database for database. The default value is `dolphinscheduler`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`DATABASE_PARAMS`**
|
||||
|
||||
This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
|
||||
|
||||
**`HADOOP_HOME`**
|
||||
|
||||
This environment variable sets `HADOOP_HOME`. The default value is `/opt/soft/hadoop`.
|
||||
|
||||
**`HADOOP_CONF_DIR`**
|
||||
|
||||
This environment variable sets `HADOOP_CONF_DIR`. The default value is `/opt/soft/hadoop/etc/hadoop`.
|
||||
|
||||
**`SPARK_HOME1`**
|
||||
|
||||
This environment variable sets `SPARK_HOME1`. The default value is `/opt/soft/spark1`.
|
||||
|
||||
**`SPARK_HOME2`**
|
||||
|
||||
This environment variable sets `SPARK_HOME2`. The default value is `/opt/soft/spark2`.
|
||||
|
||||
**`PYTHON_HOME`**
|
||||
|
||||
This environment variable sets `PYTHON_HOME`. The default value is `/usr`.
|
||||
|
||||
**`JAVA_HOME`**
|
||||
|
||||
This environment variable sets `JAVA_HOME`. The default value is `/usr/lib/jvm/java-1.8-openjdk`.
|
||||
|
||||
**`HIVE_HOME`**
|
||||
|
||||
This environment variable sets `HIVE_HOME`. The default value is `/opt/soft/hive`.
|
||||
|
||||
**`FLINK_HOME`**
|
||||
|
||||
This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/flink`.
|
||||
|
||||
**`DATAX_HOME`**
|
||||
|
||||
This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`.
|
||||
|
||||
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
|
||||
|
||||
User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler`
|
||||
|
||||
**`DOLPHINSCHEDULER_OPTS`**
|
||||
|
||||
This environment variable sets java options. The default value is empty.
|
||||
|
||||
**`RESOURCE_STORAGE_TYPE`**
|
||||
|
||||
This environment variable sets resource storage type for dolphinscheduler like `HDFS`, `S3`, `NONE`. The default value is `HDFS`.
|
||||
|
||||
**`RESOURCE_UPLOAD_PATH`**
|
||||
|
||||
This environment variable sets resource store path on HDFS/S3 for resource storage. The default value is `/dolphinscheduler`.
|
||||
|
||||
**`FS_DEFAULT_FS`**
|
||||
|
||||
This environment variable sets fs.defaultFS for resource storage like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`. The default value is `file:///`.
|
||||
|
||||
**`FS_S3A_ENDPOINT`**
|
||||
|
||||
This environment variable sets s3 endpoint for resource storage. The default value is `s3.xxx.amazonaws.com`.
|
||||
|
||||
**`FS_S3A_ACCESS_KEY`**
|
||||
|
||||
This environment variable sets s3 access key for resource storage. The default value is `xxxxxxx`.
|
||||
|
||||
**`FS_S3A_SECRET_KEY`**
|
||||
|
||||
This environment variable sets s3 secret key for resource storage. The default value is `xxxxxxx`.
|
||||
|
||||
**`ZOOKEEPER_QUORUM`**
|
||||
|
||||
This environment variable sets zookeeper quorum for `master-server` and `worker-serverr`. The default value is `127.0.0.1:2181`.
|
||||
|
||||
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`.
|
||||
|
||||
**`ZOOKEEPER_ROOT`**
|
||||
|
||||
This environment variable sets zookeeper root directory for dolphinscheduler. The default value is `/dolphinscheduler`.
|
||||
|
||||
**`MASTER_EXEC_THREADS`**
|
||||
|
||||
This environment variable sets exec thread num for `master-server`. The default value is `100`.
|
||||
|
||||
**`MASTER_EXEC_TASK_NUM`**
|
||||
|
||||
This environment variable sets exec task num for `master-server`. The default value is `20`.
|
||||
|
||||
**`MASTER_HEARTBEAT_INTERVAL`**
|
||||
|
||||
This environment variable sets heartbeat interval for `master-server`. The default value is `10`.
|
||||
|
||||
**`MASTER_TASK_COMMIT_RETRYTIMES`**
|
||||
|
||||
This environment variable sets task commit retry times for `master-server`. The default value is `5`.
|
||||
|
||||
**`MASTER_TASK_COMMIT_INTERVAL`**
|
||||
|
||||
This environment variable sets task commit interval for `master-server`. The default value is `1000`.
|
||||
|
||||
**`MASTER_MAX_CPULOAD_AVG`**
|
||||
|
||||
This environment variable sets max cpu load avg for `master-server`. The default value is `100`.
|
||||
|
||||
**`MASTER_RESERVED_MEMORY`**
|
||||
|
||||
This environment variable sets reserved memory for `master-server`. The default value is `0.1`.
|
||||
|
||||
**`MASTER_LISTEN_PORT`**
|
||||
|
||||
This environment variable sets port for `master-server`. The default value is `5678`.
|
||||
|
||||
**`WORKER_EXEC_THREADS`**
|
||||
|
||||
This environment variable sets exec thread num for `worker-server`. The default value is `100`.
|
||||
|
||||
**`WORKER_HEARTBEAT_INTERVAL`**
|
||||
|
||||
This environment variable sets heartbeat interval for `worker-server`. The default value is `10`.
|
||||
|
||||
**`WORKER_MAX_CPULOAD_AVG`**
|
||||
|
||||
This environment variable sets max cpu load avg for `worker-server`. The default value is `100`.
|
||||
|
||||
**`WORKER_RESERVED_MEMORY`**
|
||||
|
||||
This environment variable sets reserved memory for `worker-server`. The default value is `0.1`.
|
||||
|
||||
**`WORKER_LISTEN_PORT`**
|
||||
|
||||
This environment variable sets port for `worker-server`. The default value is `1234`.
|
||||
|
||||
**`WORKER_GROUPS`**
|
||||
|
||||
This environment variable sets groups for `worker-server`. The default value is `default`.
|
||||
|
||||
**`WORKER_HOST_WEIGHT`**
|
||||
|
||||
This environment variable sets weight for `worker-server`. The default value is `100`.
|
||||
|
||||
**`ALERT_LISTEN_HOST`**
|
||||
|
||||
This environment variable sets the host of `alert-server` for `worker-server`. The default value is `127.0.0.1`.
|
||||
|
||||
**`ALERT_PLUGIN_DIR`**
|
||||
|
||||
This environment variable sets the alert plugin directory for `alert-server`. The default value is `lib/plugin/alert`.
|
||||
|
||||
## Initialization scripts
|
||||
|
||||
If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`.
|
||||
|
||||
For example, to add an environment variable `API_SERVER_PORT` in `/root/start-init-conf.sh`:
|
||||
|
||||
```
|
||||
export API_SERVER_PORT=5555
|
||||
```
|
||||
|
||||
and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port:
|
||||
```
|
||||
server.port=${API_SERVER_PORT}
|
||||
```
|
||||
|
||||
`/root/start-init-conf.sh` will dynamically generate config file:
|
||||
|
||||
```sh
|
||||
echo "generate dolphinscheduler config"
|
||||
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
|
||||
eval "cat << EOF
|
||||
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
|
||||
EOF
|
||||
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
|
||||
done
|
||||
```
|
||||
|
||||
## FAQ
|
||||
|
||||
### How to stop dolphinscheduler by docker-compose?
|
||||
|
||||
Stop containers:
|
||||
|
||||
```
|
||||
docker-compose stop
|
||||
```
|
||||
|
||||
Stop containers and remove containers, networks and volumes:
|
||||
|
||||
```
|
||||
docker-compose down -v
|
||||
```
|
||||
|
||||
### How to deploy dolphinscheduler on Docker Swarm?
|
||||
|
||||
Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/))
|
||||
|
||||
Start a stack named dolphinscheduler
|
||||
|
||||
```
|
||||
docker stack deploy -c docker-stack.yml dolphinscheduler
|
||||
```
|
||||
|
||||
Stop and remove the stack named dolphinscheduler
|
||||
|
||||
```
|
||||
docker stack rm dolphinscheduler
|
||||
```
|
||||
|
||||
### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver and client of MySQL.
|
||||
>
|
||||
> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver and client:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
RUN apk add --update --no-cache mysql-client
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver and client:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql .
|
||||
```
|
||||
|
||||
4. Modify all `image` fields to `apache/dolphinscheduler:mysql` in `docker-compose.yml`
|
||||
|
||||
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
|
||||
|
||||
5. Comment the `dolphinscheduler-postgresql` block in `docker-compose.yml`
|
||||
|
||||
6. Add `dolphinscheduler-mysql` service in `docker-compose.yml` (**Optional**, you can directly use a external MySQL database)
|
||||
|
||||
7. Modify all DATABASE environments in `docker-compose.yml`
|
||||
|
||||
```
|
||||
DATABASE_TYPE: mysql
|
||||
DATABASE_DRIVER: com.mysql.jdbc.Driver
|
||||
DATABASE_HOST: dolphinscheduler-mysql
|
||||
DATABASE_PORT: 3306
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8
|
||||
```
|
||||
|
||||
> If you have added `dolphinscheduler-mysql` service in `docker-compose.yml`, just set `DATABASE_HOST` to `dolphinscheduler-mysql`
|
||||
|
||||
8. Run a dolphinscheduler (See **How to use this docker image**)
|
||||
|
||||
### How to support MySQL datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of MySQL.
|
||||
>
|
||||
> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql-driver .
|
||||
```
|
||||
|
||||
4. Modify all `image` fields to `apache/dolphinscheduler:mysql-driver` in `docker-compose.yml`
|
||||
|
||||
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
|
||||
|
||||
5. Run a dolphinscheduler (See **How to use this docker image**)
|
||||
|
||||
6. Add a MySQL datasource in `Datasource manage`
|
||||
|
||||
### How to support Oracle datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of Oracle.
|
||||
>
|
||||
> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
|
||||
|
||||
2. Create a new `Dockerfile` to add Oracle driver:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including Oracle driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:oracle-driver .
|
||||
```
|
||||
|
||||
4. Modify all `image` fields to `apache/dolphinscheduler:oracle-driver` in `docker-compose.yml`
|
||||
|
||||
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
|
||||
|
||||
5. Run a dolphinscheduler (See **How to use this docker image**)
|
||||
|
||||
6. Add a Oracle datasource in `Datasource manage`
|
||||
|
||||
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.
|
||||
|
|
@ -1,488 +0,0 @@
|
|||
## DolphinScheduler是什么?
|
||||
|
||||
一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。
|
||||
|
||||
GitHub URL: https://github.com/apache/dolphinscheduler
|
||||
|
||||
Official Website: https://dolphinscheduler.apache.org
|
||||
|
||||
|
||||
|
||||
[](README.md)
|
||||
[](README_zh_CN.md)
|
||||
|
||||
## 先决条件
|
||||
|
||||
- [Docker](https://docs.docker.com/engine/) 1.13.1+
|
||||
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
|
||||
|
||||
## 如何使用docker镜像
|
||||
|
||||
#### 以 docker-compose 的方式启动dolphinscheduler(推荐)
|
||||
|
||||
```
|
||||
$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
在`docker-compose.yml`文件中，默认的创建`Postgres`的用户、密码和数据库，默认值分别为：`root`、`root`、`dolphinscheduler`。
|
||||
|
||||
同时，默认的`Zookeeper`也会在`docker-compose.yml`文件中被创建。
|
||||
|
||||
访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler
|
||||
|
||||
默认的用户是`admin`，默认的密码是`dolphinscheduler123`
|
||||
|
||||
> **提示**: 为了在docker中快速开始，你可以创建一个名为`ds`的租户，并将这个租户`ds`关联到用户`admin`
|
||||
|
||||
#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
|
||||
|
||||
你可以指定已经存在的 **`Postgres`** 和 **`Zookeeper`** 服务. 如下:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-p 12345:12345 \
|
||||
apache/dolphinscheduler:latest all
|
||||
```
|
||||
|
||||
访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler
|
||||
|
||||
#### 或者运行dolphinscheduler中的部分服务
|
||||
|
||||
你能够运行dolphinscheduler中的部分服务。
|
||||
|
||||
* 创建一个 **本地卷** 用于资源存储，如下:
|
||||
|
||||
```
|
||||
docker volume create dolphinscheduler-resource-local
|
||||
```
|
||||
|
||||
* 启动一个 **master server**, 如下:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-master \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
apache/dolphinscheduler:latest master-server
|
||||
```
|
||||
|
||||
* 启动一个 **worker server** (包括 **logger server**), 如下:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-worker \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-e ALERT_LISTEN_HOST="dolphinscheduler-alert" \
|
||||
-v dolphinscheduler-resource-local:/dolphinscheduler \
|
||||
apache/dolphinscheduler:latest worker-server
|
||||
```
|
||||
|
||||
* 启动一个 **api server**, 如下:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-api \
|
||||
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
-v dolphinscheduler-resource-local:/dolphinscheduler \
|
||||
-p 12345:12345 \
|
||||
apache/dolphinscheduler:latest api-server
|
||||
```
|
||||
|
||||
* 启动一个 **alert server**, 如下:
|
||||
|
||||
```
|
||||
$ docker run -d --name dolphinscheduler-alert \
|
||||
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
|
||||
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
|
||||
apache/dolphinscheduler:latest alert-server
|
||||
```
|
||||
|
||||
**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。
|
||||
|
||||
## 如何构建一个docker镜像
|
||||
|
||||
你能够在类Unix系统和Windows系统中构建一个docker镜像。
|
||||
|
||||
类Unix系统, 如下:
|
||||
|
||||
```bash
|
||||
$ cd path/dolphinscheduler
|
||||
$ sh ./docker/build/hooks/build
|
||||
```
|
||||
|
||||
Windows系统, 如下:
|
||||
|
||||
```bat
|
||||
C:\dolphinscheduler>.\docker\build\hooks\build.bat
|
||||
```
|
||||
|
||||
如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。
|
||||
|
||||
## 环境变量
|
||||
|
||||
DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会使用默认值
|
||||
|
||||
**`DATABASE_TYPE`**
|
||||
|
||||
配置`database`的`TYPE`， 默认值 `postgresql`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_DRIVER`**
|
||||
|
||||
配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_HOST`**
|
||||
|
||||
配置`database`的`HOST`， 默认值 `127.0.0.1`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_PORT`**
|
||||
|
||||
配置`database`的`PORT`， 默认值 `5432`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_USERNAME`**
|
||||
|
||||
配置`database`的`USERNAME`， 默认值 `root`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_PASSWORD`**
|
||||
|
||||
配置`database`的`PASSWORD`， 默认值 `root`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_DATABASE`**
|
||||
|
||||
配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`DATABASE_PARAMS`**
|
||||
|
||||
配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`HADOOP_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`HADOOP_HOME`，默认值 `/opt/soft/hadoop`。
|
||||
|
||||
**`HADOOP_CONF_DIR`**
|
||||
|
||||
配置`dolphinscheduler`的`HADOOP_CONF_DIR`，默认值 `/opt/soft/hadoop/etc/hadoop`。
|
||||
|
||||
**`SPARK_HOME1`**
|
||||
|
||||
配置`dolphinscheduler`的`SPARK_HOME1`，默认值 `/opt/soft/spark1`。
|
||||
|
||||
**`SPARK_HOME2`**
|
||||
|
||||
配置`dolphinscheduler`的`SPARK_HOME2`，默认值 `/opt/soft/spark2`。
|
||||
|
||||
**`PYTHON_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr`。
|
||||
|
||||
**`JAVA_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`JAVA_HOME`，默认值 `/usr/lib/jvm/java-1。8-openjdk`。
|
||||
|
||||
**`HIVE_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`HIVE_HOME`，默认值 `/opt/soft/hive`。
|
||||
|
||||
**`FLINK_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`FLINK_HOME`，默认值 `/opt/soft/flink`。
|
||||
|
||||
**`DATAX_HOME`**
|
||||
|
||||
配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。
|
||||
|
||||
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
|
||||
|
||||
用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。
|
||||
|
||||
**`DOLPHINSCHEDULER_OPTS`**
|
||||
|
||||
配置`dolphinscheduler`的`java options`，默认值 `""`、
|
||||
|
||||
**`RESOURCE_STORAGE_TYPE`**
|
||||
|
||||
配置`dolphinscheduler`的资源存储类型，可选项为 `HDFS`、`S3`、`NONE`，默认值 `HDFS`。
|
||||
|
||||
**`RESOURCE_UPLOAD_PATH`**
|
||||
|
||||
配置`HDFS/S3`上的资源存储路径，默认值 `/dolphinscheduler`。
|
||||
|
||||
**`FS_DEFAULT_FS`**
|
||||
|
||||
配置资源存储的文件系统协议，如 `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`，默认值 `file:///`。
|
||||
|
||||
**`FS_S3A_ENDPOINT`**
|
||||
|
||||
当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的访问路径，默认值 `s3.xxx.amazonaws.com`。
|
||||
|
||||
**`FS_S3A_ACCESS_KEY`**
|
||||
|
||||
当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的`s3 access key`，默认值 `xxxxxxx`。
|
||||
|
||||
**`FS_S3A_SECRET_KEY`**
|
||||
|
||||
当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的`s3 secret key`，默认值 `xxxxxxx`。
|
||||
|
||||
**`ZOOKEEPER_QUORUM`**
|
||||
|
||||
配置`master-server`和`worker-serverr`的`Zookeeper`地址, 默认值 `127.0.0.1:2181`。
|
||||
|
||||
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
|
||||
|
||||
**`ZOOKEEPER_ROOT`**
|
||||
|
||||
配置`dolphinscheduler`在`zookeeper`中数据存储的根目录，默认值 `/dolphinscheduler`。
|
||||
|
||||
**`MASTER_EXEC_THREADS`**
|
||||
|
||||
配置`master-server`中的执行线程数量，默认值 `100`。
|
||||
|
||||
**`MASTER_EXEC_TASK_NUM`**
|
||||
|
||||
配置`master-server`中的执行任务数量，默认值 `20`。
|
||||
|
||||
**`MASTER_HEARTBEAT_INTERVAL`**
|
||||
|
||||
配置`master-server`中的心跳交互时间，默认值 `10`。
|
||||
|
||||
**`MASTER_TASK_COMMIT_RETRYTIMES`**
|
||||
|
||||
配置`master-server`中的任务提交重试次数，默认值 `5`。
|
||||
|
||||
**`MASTER_TASK_COMMIT_INTERVAL`**
|
||||
|
||||
配置`master-server`中的任务提交交互时间，默认值 `1000`。
|
||||
|
||||
**`MASTER_MAX_CPULOAD_AVG`**
|
||||
|
||||
配置`master-server`中的CPU中的`load average`值，默认值 `100`。
|
||||
|
||||
**`MASTER_RESERVED_MEMORY`**
|
||||
|
||||
配置`master-server`的保留内存，默认值 `0.1`。
|
||||
|
||||
**`MASTER_LISTEN_PORT`**
|
||||
|
||||
配置`master-server`的端口，默认值 `5678`。
|
||||
|
||||
**`WORKER_EXEC_THREADS`**
|
||||
|
||||
配置`worker-server`中的执行线程数量，默认值 `100`。
|
||||
|
||||
**`WORKER_HEARTBEAT_INTERVAL`**
|
||||
|
||||
配置`worker-server`中的心跳交互时间，默认值 `10`。
|
||||
|
||||
**`WORKER_MAX_CPULOAD_AVG`**
|
||||
|
||||
配置`worker-server`中的CPU中的最大`load average`值，默认值 `100`。
|
||||
|
||||
**`WORKER_RESERVED_MEMORY`**
|
||||
|
||||
配置`worker-server`的保留内存，默认值 `0.1`。
|
||||
|
||||
**`WORKER_LISTEN_PORT`**
|
||||
|
||||
配置`worker-server`的端口，默认值 `1234`。
|
||||
|
||||
**`WORKER_GROUPS`**
|
||||
|
||||
配置`worker-server`的分组，默认值 `default`。
|
||||
|
||||
**`WORKER_HOST_WEIGHT`**
|
||||
|
||||
配置`worker-server`的权重，默认之`100`。
|
||||
|
||||
**`ALERT_LISTEN_HOST`**
|
||||
|
||||
配置`worker-server`的告警主机，即`alert-server`的主机名，默认值 `127.0.0.1`。
|
||||
|
||||
**`ALERT_PLUGIN_DIR`**
|
||||
|
||||
配置`alert-server`的告警插件目录，默认值 `lib/plugin/alert`。
|
||||
|
||||
## 初始化脚本
|
||||
|
||||
如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量，你可以在`/root/start-init-conf.sh`文件中进行修改，同时如果涉及到配置文件的修改，请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件
|
||||
|
||||
例如，在`/root/start-init-conf.sh`添加一个环境变量`API_SERVER_PORT`：
|
||||
|
||||
```
|
||||
export API_SERVER_PORT=5555
|
||||
```
|
||||
|
||||
当添加以上环境变量后，你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置:
|
||||
```
|
||||
server.port=${API_SERVER_PORT}
|
||||
```
|
||||
|
||||
`/root/start-init-conf.sh`将根据模板文件动态的生成配置文件：
|
||||
|
||||
```sh
|
||||
echo "generate dolphinscheduler config"
|
||||
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
|
||||
eval "cat << EOF
|
||||
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
|
||||
EOF
|
||||
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
|
||||
done
|
||||
```
|
||||
|
||||
## FAQ
|
||||
|
||||
### 如何通过 docker-compose 停止 dolphinscheduler？
|
||||
|
||||
停止所有容器:
|
||||
|
||||
```
|
||||
docker-compose stop
|
||||
```
|
||||
|
||||
停止所有容器并移除所有容器，网络和存储卷:
|
||||
|
||||
```
|
||||
docker-compose down -v
|
||||
```
|
||||
|
||||
### 如何在 Docker Swarm 上部署 dolphinscheduler？
|
||||
|
||||
假设 Docker Swarm 集群已经部署（如果还没有创建 Docker Swarm 集群，请参考 [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)）
|
||||
|
||||
启动名为 dolphinscheduler 的 stack
|
||||
|
||||
```
|
||||
docker stack deploy -c docker-stack.yml dolphinscheduler
|
||||
```
|
||||
|
||||
启动并移除名为 dolphinscheduler 的 stack
|
||||
|
||||
```
|
||||
docker stack rm dolphinscheduler
|
||||
```
|
||||
|
||||
### 如何用 MySQL 替代 PostgreSQL 作为 DolphinScheduler 的数据库？
|
||||
|
||||
> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包和客户端.
|
||||
>
|
||||
> 如果你要使用 MySQL, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
|
||||
|
||||
1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`)
|
||||
|
||||
2. 创建一个新的 `Dockerfile`，用于添加 MySQL 的驱动包和客户端:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
RUN apk add --update --no-cache mysql-client
|
||||
```
|
||||
|
||||
3. 构建一个包含 MySQL 的驱动包和客户端的新镜像:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql .
|
||||
```
|
||||
|
||||
4. 修改 `docker-compose.yml` 文件中的所有 image 字段为 `apache/dolphinscheduler:mysql`
|
||||
|
||||
> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml`
|
||||
|
||||
5. 注释 `docker-compose.yml` 文件中的 `dolphinscheduler-postgresql` 块
|
||||
|
||||
6. 在 `docker-compose.yml` 文件中添加 `dolphinscheduler-mysql` 服务（**可选**，你可以直接使用一个外部的 MySQL 数据库）
|
||||
|
||||
7. 修改 `docker-compose.yml` 文件中的所有 DATABASE 环境变量
|
||||
|
||||
```
|
||||
DATABASE_TYPE: mysql
|
||||
DATABASE_DRIVER: com.mysql.jdbc.Driver
|
||||
DATABASE_HOST: dolphinscheduler-mysql
|
||||
DATABASE_PORT: 3306
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8
|
||||
```
|
||||
|
||||
> 如果你已经添加了 `dolphinscheduler-mysql` 服务，设置 `DATABASE_HOST` 为 `dolphinscheduler-mysql` 即可
|
||||
|
||||
8. 运行 dolphinscheduler (详见**如何使用docker镜像**)
|
||||
|
||||
### 如何在数据源中心支持 MySQL 数据源？
|
||||
|
||||
> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包.
|
||||
>
|
||||
> 如果你要添加 MySQL 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
|
||||
|
||||
1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`)
|
||||
|
||||
2. 创建一个新的 `Dockerfile`，用于添加 MySQL 驱动包:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. 构建一个包含 MySQL 驱动包的新镜像:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql-driver .
|
||||
```
|
||||
|
||||
4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:mysql-driver`
|
||||
|
||||
> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml`
|
||||
|
||||
5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
|
||||
|
||||
6. 在数据源中心添加一个 MySQL 数据源
|
||||
|
||||
### 如何在数据源中心支持 Oracle 数据源？
|
||||
|
||||
> 由于商业许可证的原因，我们不能直接使用 Oracle 的驱动包.
|
||||
>
|
||||
> 如果你要添加 Oracle 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
|
||||
|
||||
1. 下载 Oracle 驱动包 [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
|
||||
|
||||
2. 创建一个新的 `Dockerfile`，用于添加 Oracle 驱动包:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. 构建一个包含 Oracle 驱动包的新镜像:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:oracle-driver .
|
||||
```
|
||||
|
||||
4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:oracle-driver`
|
||||
|
||||
> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml`
|
||||
|
||||
5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
|
||||
|
||||
6. 在数据源中心添加一个 Oracle 数据源
|
||||
|
||||
更多信息请查看 [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) 文档.
|
||||
|
|
@ -41,12 +41,21 @@ server.jetty.max-http-form-post-size=5000000
|
|||
# messages encoding
|
||||
spring.messages.encoding=UTF-8
|
||||
|
||||
# i18n classpath folder , file prefix messages， if have many files, use "," seperator
|
||||
# i18n classpath folder, file prefix messages. if have many files, use "," seperator
|
||||
spring.messages.basename=i18n/messages
|
||||
|
||||
# Authentication types (supported types: PASSWORD)
|
||||
security.authentication.type=PASSWORD
|
||||
|
||||
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
|
||||
# global max request number per second
|
||||
# default tenant-level max request number
|
||||
#traffic.control.global.switch=true
|
||||
#traffic.control.max.global.qps.rate=500
|
||||
#traffic.control.tenant.switch=true
|
||||
#traffic.control.default.tenant.qps.rate=10
|
||||
#traffic.control.customize.tenant.qps.rate={'tenant1':11,'tenant2':20}
|
||||
|
||||
#============================================================================
|
||||
# LDAP Config
|
||||
# mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/
|
||||
|
|
|
|||
|
|
@ -15,34 +15,37 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# resource storage type : HDFS, S3, NONE
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
data.basedir.path=${DATA_BASEDIR_PATH}
|
||||
|
||||
# resource storage type: HDFS, S3, NONE
|
||||
resource.storage.type=${RESOURCE_STORAGE_TYPE}
|
||||
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resource.upload.path=${RESOURCE_UPLOAD_PATH}
|
||||
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
|
||||
|
||||
# whether kerberos starts
|
||||
hadoop.security.authentication.startup.state=false
|
||||
# whether to startup kerberos
|
||||
hadoop.security.authentication.startup.state=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE}
|
||||
|
||||
# java.security.krb5.conf path
|
||||
java.security.krb5.conf.path=/opt/krb5.conf
|
||||
java.security.krb5.conf.path=${JAVA_SECURITY_KRB5_CONF_PATH}
|
||||
|
||||
# login user from keytab username
|
||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
||||
login.user.keytab.username=${LOGIN_USER_KEYTAB_USERNAME}
|
||||
|
||||
# login user from keytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
login.user.keytab.path=${LOGIN_USER_KEYTAB_PATH}
|
||||
|
||||
#resource.view.suffixs
|
||||
# kerberos expire time, the unit is hour
|
||||
kerberos.expire.time=${KERBEROS_EXPIRE_TIME}
|
||||
|
||||
# resource view suffixs
|
||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
||||
|
||||
# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path
|
||||
hdfs.root.user=hdfs
|
||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
||||
hdfs.root.user=${HDFS_ROOT_USER}
|
||||
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
fs.defaultFS=${FS_DEFAULT_FS}
|
||||
|
||||
# if resource.storage.type=S3, s3 endpoint
|
||||
|
|
@ -54,25 +57,32 @@ fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
|
|||
# if resource.storage.type=S3, s3 secret key
|
||||
fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
|
||||
|
||||
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
# resourcemanager port, the default value is 8088 if not specified
|
||||
resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT}
|
||||
|
||||
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.
|
||||
yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s
|
||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS}
|
||||
|
||||
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000)
|
||||
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
|
||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
||||
yarn.application.status.address=${YARN_APPLICATION_STATUS_ADDRESS}
|
||||
|
||||
# system env path, If you want to set your own path, you need to set this env file to an absolute path
|
||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
||||
development.state=false
|
||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
||||
yarn.job.history.status.address=${YARN_JOB_HISTORY_STATUS_ADDRESS}
|
||||
|
||||
# kerberos tgt expire time, unit is hours
|
||||
kerberos.expire.time=2
|
||||
# datasource encryption enable
|
||||
datasource.encryption.enable=${DATASOURCE_ENCRYPTION_ENABLE}
|
||||
|
||||
# datasource encryption salt
|
||||
datasource.encryption.enable=false
|
||||
datasource.encryption.salt=!@#$%^&*
|
||||
datasource.encryption.salt=${DATASOURCE_ENCRYPTION_SALT}
|
||||
|
||||
# Network IP gets priority, default inner outer
|
||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
|
||||
sudo.enable=${SUDO_ENABLE}
|
||||
|
||||
# network IP gets priority, default: inner outer
|
||||
#dolphin.scheduler.network.priority.strategy=default
|
||||
|
||||
# system env path
|
||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
||||
|
||||
# development state
|
||||
development.state=false
|
||||
|
|
|
|||
|
|
@ -24,14 +24,14 @@ spring.datasource.password=${DATABASE_PASSWORD}
|
|||
# postgresql
|
||||
#spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
#spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
|
||||
#spring.datasource.username=test
|
||||
#spring.datasource.password=test
|
||||
#spring.datasource.username=root
|
||||
#spring.datasource.password=root
|
||||
|
||||
# mysql
|
||||
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
|
||||
#spring.datasource.username=xxxx
|
||||
#spring.datasource.password=xxxx
|
||||
#spring.datasource.username=ds_user
|
||||
#spring.datasource.password=dolphinscheduler
|
||||
|
||||
# connection configuration
|
||||
#spring.datasource.initialSize=5
|
||||
|
|
|
|||
|
|
@ -15,29 +15,32 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# master execute thread number
|
||||
# master listen port
|
||||
#master.listen.port=5678
|
||||
|
||||
# master execute thread number to limit process instances in parallel
|
||||
master.exec.threads=${MASTER_EXEC_THREADS}
|
||||
|
||||
# master execute task number in parallel
|
||||
# master execute task number in parallel per process instance
|
||||
master.exec.task.num=${MASTER_EXEC_TASK_NUM}
|
||||
|
||||
# master dispatch task number
|
||||
#master.dispatch.task.num=3
|
||||
# master dispatch task number per batch
|
||||
master.dispatch.task.num=${MASTER_DISPATCH_TASK_NUM}
|
||||
|
||||
# master heartbeat interval
|
||||
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight
|
||||
master.host.selector=${MASTER_HOST_SELECTOR}
|
||||
|
||||
# master heartbeat interval, the unit is second
|
||||
master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL}
|
||||
|
||||
# master commit task retry times
|
||||
master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES}
|
||||
|
||||
# master commit task interval
|
||||
# master commit task interval, the unit is millisecond
|
||||
master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL}
|
||||
|
||||
# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
|
||||
# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
|
||||
master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG}
|
||||
|
||||
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
|
||||
# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
|
||||
master.reserved.memory=${MASTER_RESERVED_MEMORY}
|
||||
|
||||
# master listen port
|
||||
master.listen.port=${MASTER_LISTEN_PORT}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,13 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# worker execute thread number
|
||||
# worker listener port
|
||||
#worker.listen.port=1234
|
||||
|
||||
# worker execute thread number to limit task instances in parallel
|
||||
worker.exec.threads=${WORKER_EXEC_THREADS}
|
||||
|
||||
# worker heartbeat interval
|
||||
# worker heartbeat interval, the unit is second
|
||||
worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL}
|
||||
|
||||
# worker host weight to dispatch tasks, default value 100
|
||||
|
|
@ -27,17 +30,14 @@ worker.host.weight=${WORKER_HOST_WEIGHT}
|
|||
# worker tenant auto create
|
||||
worker.tenant.auto.create=true
|
||||
|
||||
# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2
|
||||
# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
|
||||
worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG}
|
||||
|
||||
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
|
||||
# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
|
||||
worker.reserved.memory=${WORKER_RESERVED_MEMORY}
|
||||
|
||||
# worker listener port
|
||||
worker.listen.port=${WORKER_LISTEN_PORT}
|
||||
|
||||
# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test'
|
||||
# default worker groups separated by comma, like 'worker.groups=default,test'
|
||||
worker.groups=${WORKER_GROUPS}
|
||||
|
||||
# alert server listener host
|
||||
# alert server listen host
|
||||
alert.listen.host=${ALERT_LISTEN_HOST}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ echo "init env variables"
|
|||
# Define parameters default value
|
||||
|
||||
#============================================================================
|
||||
# Database Source
|
||||
# Database
|
||||
#============================================================================
|
||||
export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
|
||||
export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
|
||||
|
|
@ -34,64 +34,91 @@ export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
|
|||
export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
|
||||
export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
|
||||
|
||||
#============================================================================
|
||||
# ZooKeeper
|
||||
#============================================================================
|
||||
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
|
||||
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
|
||||
|
||||
#============================================================================
|
||||
# Common
|
||||
#============================================================================
|
||||
# dolphinscheduler env
|
||||
export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
|
||||
export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
|
||||
export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
|
||||
export PYTHON_HOME=${PYTHON_HOME:-"/usr"}
|
||||
export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"}
|
||||
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
|
||||
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
|
||||
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
|
||||
# common env
|
||||
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
|
||||
# common opts
|
||||
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
|
||||
# common env
|
||||
export DATA_BASEDIR_PATH=${DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
|
||||
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
|
||||
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
|
||||
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
|
||||
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
|
||||
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
|
||||
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
|
||||
|
||||
#============================================================================
|
||||
# Zookeeper
|
||||
#============================================================================
|
||||
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
|
||||
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
|
||||
export HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE:-"false"}
|
||||
export JAVA_SECURITY_KRB5_CONF_PATH=${JAVA_SECURITY_KRB5_CONF_PATH:-"/opt/krb5.conf"}
|
||||
export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM"}
|
||||
export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"}
|
||||
export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"}
|
||||
export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"}
|
||||
export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"}
|
||||
export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""}
|
||||
export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:%s/ws/v1/cluster/apps/%s"}
|
||||
export YARN_JOB_HISTORY_STATUS_ADDRESS=${YARN_JOB_HISTORY_STATUS_ADDRESS:-"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"}
|
||||
export DATASOURCE_ENCRYPTION_ENABLE=${DATASOURCE_ENCRYPTION_ENABLE:-"false"}
|
||||
export DATASOURCE_ENCRYPTION_SALT=${DATASOURCE_ENCRYPTION_SALT:-"!@#$%^&*"}
|
||||
export SUDO_ENABLE=${SUDO_ENABLE:-"true"}
|
||||
# dolphinscheduler env
|
||||
export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
|
||||
export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
|
||||
export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
|
||||
export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"}
|
||||
export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"}
|
||||
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
|
||||
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
|
||||
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
|
||||
|
||||
#============================================================================
|
||||
# Master Server
|
||||
#============================================================================
|
||||
export MASTER_SERVER_OPTS=${MASTER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
|
||||
export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"}
|
||||
export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"}
|
||||
export MASTER_DISPATCH_TASK_NUM=${MASTER_DISPATCH_TASK_NUM:-"3"}
|
||||
export MASTER_HOST_SELECTOR=${MASTER_HOST_SELECTOR:-"LowerWeight"}
|
||||
export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"}
|
||||
export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"}
|
||||
export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"}
|
||||
export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"100"}
|
||||
export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.1"}
|
||||
export MASTER_LISTEN_PORT=${MASTER_LISTEN_PORT:-"5678"}
|
||||
export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"-1"}
|
||||
export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.3"}
|
||||
|
||||
#============================================================================
|
||||
# Worker Server
|
||||
#============================================================================
|
||||
export WORKER_SERVER_OPTS=${WORKER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
|
||||
export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"}
|
||||
export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"}
|
||||
export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"}
|
||||
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"}
|
||||
export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"}
|
||||
export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
|
||||
export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"}
|
||||
export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"}
|
||||
export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"-1"}
|
||||
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.3"}
|
||||
export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
|
||||
export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"localhost"}
|
||||
|
||||
#============================================================================
|
||||
# Alert Server
|
||||
#============================================================================
|
||||
export ALERT_SERVER_OPTS=${ALERT_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
|
||||
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"}
|
||||
|
||||
#============================================================================
|
||||
# Api Server
|
||||
#============================================================================
|
||||
export API_SERVER_OPTS=${API_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
|
||||
|
||||
#============================================================================
|
||||
# Logger Server
|
||||
#============================================================================
|
||||
export LOGGER_SERVER_OPTS=${LOGGER_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
|
||||
|
||||
echo "generate dolphinscheduler config"
|
||||
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
|
||||
eval "cat << EOF
|
||||
|
|
@ -100,9 +127,9 @@ EOF
|
|||
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
|
||||
done
|
||||
|
||||
# generate dolphinscheduler env only in docker
|
||||
# generate dolphinscheduler env
|
||||
DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
|
||||
if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
|
||||
if [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
|
||||
eval "cat << EOF
|
||||
$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ export LOGGER_START_ENABLED=false
|
|||
|
||||
# wait database
|
||||
waitDatabase() {
|
||||
echo "test ${DATABASE_TYPE} service"
|
||||
echo "try to connect ${DATABASE_TYPE} ..."
|
||||
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
|
||||
local counter=$((counter+1))
|
||||
if [ $counter == 30 ]; then
|
||||
|
|
@ -37,21 +37,7 @@ waitDatabase() {
|
|||
echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "connect ${DATABASE_TYPE} service"
|
||||
if [ ${DATABASE_TYPE} = "mysql" ]; then
|
||||
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
|
||||
if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then
|
||||
echo "Error: Can't connect to database...${v}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
|
||||
if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
|
||||
echo "Error: Can't connect to database...${v}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "${DATABASE_TYPE} connection is ok"
|
||||
}
|
||||
|
||||
# init database
|
||||
|
|
@ -60,38 +46,9 @@ initDatabase() {
|
|||
${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
|
||||
}
|
||||
|
||||
# check ds version
|
||||
checkDSVersion() {
|
||||
if [ ${DATABASE_TYPE} = "mysql" ]; then
|
||||
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "SELECT * FROM public.t_ds_version" 2>/dev/null)
|
||||
else
|
||||
v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "SELECT * FROM public.t_ds_version" 2>/dev/null)
|
||||
fi
|
||||
if [ -n "$v" ]; then
|
||||
echo "ds version: $v"
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# check init database
|
||||
checkInitDatabase() {
|
||||
echo "check init database"
|
||||
while ! checkDSVersion; do
|
||||
local counter=$((counter+1))
|
||||
if [ $counter == 30 ]; then
|
||||
echo "Error: Couldn't check init database."
|
||||
exit 1
|
||||
fi
|
||||
echo "Trying to check init database. Attempt $counter."
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
# wait zk
|
||||
waitZK() {
|
||||
echo "connect remote zookeeper"
|
||||
echo "try to connect zookeeper ..."
|
||||
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
|
||||
while ! nc -z ${line%:*} ${line#*:}; do
|
||||
local counter=$((counter+1))
|
||||
|
|
@ -103,6 +60,7 @@ waitZK() {
|
|||
sleep 5
|
||||
done
|
||||
done
|
||||
echo "zookeeper connection is ok"
|
||||
}
|
||||
|
||||
# print usage
|
||||
|
|
@ -150,7 +108,6 @@ case "$1" in
|
|||
;;
|
||||
(alert-server)
|
||||
waitDatabase
|
||||
checkInitDatabase
|
||||
export ALERT_START_ENABLED=true
|
||||
;;
|
||||
(help)
|
||||
|
|
|
|||
|
|
@ -15,21 +15,28 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
echo "------ dolphinscheduler check - server - status -------"
|
||||
sleep 60
|
||||
server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
|
||||
if [ $server_num -eq 5 ]
|
||||
then
|
||||
echo "Server all start successfully"
|
||||
else
|
||||
echo "Server start failed "$server_num
|
||||
exit 1
|
||||
fi
|
||||
ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
|
||||
if [ $ready -eq 1 ]
|
||||
then
|
||||
echo "Servers is ready"
|
||||
else
|
||||
echo "Servers is not ready"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "------- check server status -------"
|
||||
cd $(dirname $0)
|
||||
until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do
|
||||
counter=$((counter+1))
|
||||
if [ $counter -eq 30 ]; then
|
||||
echo "error: all servers start failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "waiting for all servers started ..."
|
||||
sleep 5
|
||||
done
|
||||
echo "all servers start successfully"
|
||||
|
||||
counter=0
|
||||
until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do
|
||||
counter=$((counter+1))
|
||||
if [ $counter -eq 30 ]; then
|
||||
echo "error: api server is not ready"
|
||||
exit 1
|
||||
fi
|
||||
echo "waiting for api server ready ..."
|
||||
sleep 5
|
||||
done
|
||||
echo "api server is ready"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#============================================================================
|
||||
# Database
|
||||
#============================================================================
|
||||
# postgresql
|
||||
DATABASE_TYPE=postgresql
|
||||
DATABASE_DRIVER=org.postgresql.Driver
|
||||
DATABASE_HOST=dolphinscheduler-postgresql
|
||||
DATABASE_PORT=5432
|
||||
DATABASE_USERNAME=root
|
||||
DATABASE_PASSWORD=root
|
||||
DATABASE_DATABASE=dolphinscheduler
|
||||
DATABASE_PARAMS=characterEncoding=utf8
|
||||
# mysql
|
||||
# DATABASE_TYPE=mysql
|
||||
# DATABASE_DRIVER=com.mysql.jdbc.Driver
|
||||
# DATABASE_HOST=dolphinscheduler-mysql
|
||||
# DATABASE_PORT=3306
|
||||
# DATABASE_USERNAME=root
|
||||
# DATABASE_PASSWORD=root
|
||||
# DATABASE_DATABASE=dolphinscheduler
|
||||
# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
|
||||
|
||||
#============================================================================
|
||||
# ZooKeeper
|
||||
#============================================================================
|
||||
ZOOKEEPER_QUORUM=dolphinscheduler-zookeeper:2181
|
||||
ZOOKEEPER_ROOT=/dolphinscheduler
|
||||
|
||||
#============================================================================
|
||||
# Common
|
||||
#============================================================================
|
||||
# common opts
|
||||
DOLPHINSCHEDULER_OPTS=
|
||||
# common env
|
||||
DATA_BASEDIR_PATH=/tmp/dolphinscheduler
|
||||
RESOURCE_STORAGE_TYPE=HDFS
|
||||
RESOURCE_UPLOAD_PATH=/dolphinscheduler
|
||||
FS_DEFAULT_FS=file:///
|
||||
FS_S3A_ENDPOINT=s3.xxx.amazonaws.com
|
||||
FS_S3A_ACCESS_KEY=xxxxxxx
|
||||
FS_S3A_SECRET_KEY=xxxxxxx
|
||||
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false
|
||||
JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf
|
||||
LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM
|
||||
LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab
|
||||
KERBEROS_EXPIRE_TIME=2
|
||||
HDFS_ROOT_USER=hdfs
|
||||
RESOURCE_MANAGER_HTTPADDRESS_PORT=8088
|
||||
YARN_RESOURCEMANAGER_HA_RM_IDS=
|
||||
YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s
|
||||
YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
|
||||
DATASOURCE_ENCRYPTION_ENABLE=false
|
||||
DATASOURCE_ENCRYPTION_SALT=!@#$%^&*
|
||||
SUDO_ENABLE=true
|
||||
# dolphinscheduler env
|
||||
HADOOP_HOME=/opt/soft/hadoop
|
||||
HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
|
||||
SPARK_HOME1=/opt/soft/spark1
|
||||
SPARK_HOME2=/opt/soft/spark2
|
||||
PYTHON_HOME=/usr/bin/python
|
||||
JAVA_HOME=/usr/local/openjdk-8
|
||||
HIVE_HOME=/opt/soft/hive
|
||||
FLINK_HOME=/opt/soft/flink
|
||||
DATAX_HOME=/opt/soft/datax
|
||||
|
||||
#============================================================================
|
||||
# Master Server
|
||||
#============================================================================
|
||||
MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
|
||||
MASTER_EXEC_THREADS=100
|
||||
MASTER_EXEC_TASK_NUM=20
|
||||
MASTER_DISPATCH_TASK_NUM=3
|
||||
MASTER_HOST_SELECTOR=LowerWeight
|
||||
MASTER_HEARTBEAT_INTERVAL=10
|
||||
MASTER_TASK_COMMIT_RETRYTIMES=5
|
||||
MASTER_TASK_COMMIT_INTERVAL=1000
|
||||
MASTER_MAX_CPULOAD_AVG=-1
|
||||
MASTER_RESERVED_MEMORY=0.3
|
||||
|
||||
#============================================================================
|
||||
# Worker Server
|
||||
#============================================================================
|
||||
WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
|
||||
WORKER_EXEC_THREADS=100
|
||||
WORKER_HEARTBEAT_INTERVAL=10
|
||||
WORKER_HOST_WEIGHT=100
|
||||
WORKER_MAX_CPULOAD_AVG=-1
|
||||
WORKER_RESERVED_MEMORY=0.3
|
||||
WORKER_GROUPS=default
|
||||
ALERT_LISTEN_HOST=dolphinscheduler-alert
|
||||
|
||||
#============================================================================
|
||||
# Alert Server
|
||||
#============================================================================
|
||||
ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
|
||||
ALERT_PLUGIN_DIR=lib/plugin/alert
|
||||
|
||||
#============================================================================
|
||||
# Api Server
|
||||
#============================================================================
|
||||
API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
|
||||
|
||||
#============================================================================
|
||||
# Logger Server
|
||||
#============================================================================
|
||||
LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
|
||||
|
|
@ -19,10 +19,7 @@ version: "3.1"
|
|||
services:
|
||||
|
||||
dolphinscheduler-postgresql:
|
||||
image: bitnami/postgresql:latest
|
||||
container_name: dolphinscheduler-postgresql
|
||||
ports:
|
||||
- 5432:5432
|
||||
image: bitnami/postgresql:11.11.0
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
POSTGRESQL_USERNAME: root
|
||||
|
|
@ -30,16 +27,12 @@ services:
|
|||
POSTGRESQL_DATABASE: dolphinscheduler
|
||||
volumes:
|
||||
- dolphinscheduler-postgresql:/bitnami/postgresql
|
||||
- dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-zookeeper:
|
||||
image: bitnami/zookeeper:latest
|
||||
container_name: dolphinscheduler-zookeeper
|
||||
ports:
|
||||
- 2181:2181
|
||||
image: bitnami/zookeeper:3.6.2
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
ALLOW_ANONYMOUS_LOGIN: "yes"
|
||||
|
|
@ -51,26 +44,13 @@ services:
|
|||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-api:
|
||||
image: apache/dolphinscheduler:latest
|
||||
container_name: dolphinscheduler-api
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: api-server
|
||||
ports:
|
||||
- 12345:12345
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
RESOURCE_STORAGE_TYPE: HDFS
|
||||
RESOURCE_UPLOAD_PATH: /dolphinscheduler
|
||||
FS_DEFAULT_FS: file:///
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
|
||||
interval: 30s
|
||||
|
|
@ -81,29 +61,18 @@ services:
|
|||
- dolphinscheduler-zookeeper
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-alert:
|
||||
image: apache/dolphinscheduler:latest
|
||||
container_name: dolphinscheduler-alert
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: alert-server
|
||||
ports:
|
||||
- 50052:50052
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
ALERT_PLUGIN_DIR: lib/plugin/alert
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
|
||||
interval: 30s
|
||||
|
|
@ -118,31 +87,11 @@ services:
|
|||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-master:
|
||||
image: apache/dolphinscheduler:latest
|
||||
container_name: dolphinscheduler-master
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: master-server
|
||||
ports:
|
||||
- 5678:5678
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
MASTER_EXEC_THREADS: "100"
|
||||
MASTER_EXEC_TASK_NUM: "20"
|
||||
MASTER_HEARTBEAT_INTERVAL: "10"
|
||||
MASTER_TASK_COMMIT_RETRYTIMES: "5"
|
||||
MASTER_TASK_COMMIT_INTERVAL: "1000"
|
||||
MASTER_MAX_CPULOAD_AVG: "100"
|
||||
MASTER_RESERVED_MEMORY: "0.1"
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
|
||||
interval: 30s
|
||||
|
|
@ -153,49 +102,17 @@ services:
|
|||
- dolphinscheduler-zookeeper
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-worker:
|
||||
image: apache/dolphinscheduler:latest
|
||||
container_name: dolphinscheduler-worker
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: worker-server
|
||||
ports:
|
||||
- 1234:1234
|
||||
- 50051:50051
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
WORKER_EXEC_THREADS: "100"
|
||||
WORKER_HEARTBEAT_INTERVAL: "10"
|
||||
WORKER_MAX_CPULOAD_AVG: "100"
|
||||
WORKER_RESERVED_MEMORY: "0.1"
|
||||
WORKER_GROUPS: "default"
|
||||
WORKER_HOST_WEIGHT: "100"
|
||||
ALERT_LISTEN_HOST: dolphinscheduler-alert
|
||||
HADOOP_HOME: "/opt/soft/hadoop"
|
||||
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
|
||||
SPARK_HOME1: "/opt/soft/spark1"
|
||||
SPARK_HOME2: "/opt/soft/spark2"
|
||||
#PYTHON_HOME: "/opt/soft/python"
|
||||
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
|
||||
HIVE_HOME: "/opt/soft/hive"
|
||||
FLINK_HOME: "/opt/soft/flink"
|
||||
DATAX_HOME: "/opt/soft/datax"
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
RESOURCE_STORAGE_TYPE: HDFS
|
||||
RESOURCE_UPLOAD_PATH: /dolphinscheduler
|
||||
FS_DEFAULT_FS: file:///
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
|
||||
interval: 30s
|
||||
|
|
@ -207,6 +124,7 @@ services:
|
|||
volumes:
|
||||
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
|
|
@ -218,8 +136,8 @@ networks:
|
|||
|
||||
volumes:
|
||||
dolphinscheduler-postgresql:
|
||||
dolphinscheduler-postgresql-initdb:
|
||||
dolphinscheduler-zookeeper:
|
||||
dolphinscheduler-worker-data:
|
||||
dolphinscheduler-logs:
|
||||
dolphinscheduler-shared-local:
|
||||
dolphinscheduler-resource-local:
|
||||
|
|
@ -19,9 +19,7 @@ version: "3.1"
|
|||
services:
|
||||
|
||||
dolphinscheduler-postgresql:
|
||||
image: bitnami/postgresql:latest
|
||||
ports:
|
||||
- 5432:5432
|
||||
image: bitnami/postgresql:11.11.0
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
POSTGRESQL_USERNAME: root
|
||||
|
|
@ -36,9 +34,7 @@ services:
|
|||
replicas: 1
|
||||
|
||||
dolphinscheduler-zookeeper:
|
||||
image: bitnami/zookeeper:latest
|
||||
ports:
|
||||
- 2181:2181
|
||||
image: bitnami/zookeeper:3.6.2
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
ALLOW_ANONYMOUS_LOGIN: "yes"
|
||||
|
|
@ -52,25 +48,13 @@ services:
|
|||
replicas: 1
|
||||
|
||||
dolphinscheduler-api:
|
||||
image: apache/dolphinscheduler:latest
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: api-server
|
||||
ports:
|
||||
- 12345:12345
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
RESOURCE_STORAGE_TYPE: HDFS
|
||||
RESOURCE_UPLOAD_PATH: /dolphinscheduler
|
||||
FS_DEFAULT_FS: file:///
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
|
||||
interval: 30s
|
||||
|
|
@ -78,6 +62,8 @@ services:
|
|||
retries: 3
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
deploy:
|
||||
|
|
@ -85,22 +71,11 @@ services:
|
|||
replicas: 1
|
||||
|
||||
dolphinscheduler-alert:
|
||||
image: apache/dolphinscheduler:latest
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: alert-server
|
||||
ports:
|
||||
- 50052:50052
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
ALERT_PLUGIN_DIR: lib/plugin/alert
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
|
||||
interval: 30s
|
||||
|
|
@ -115,30 +90,11 @@ services:
|
|||
replicas: 1
|
||||
|
||||
dolphinscheduler-master:
|
||||
image: apache/dolphinscheduler:latest
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: master-server
|
||||
ports:
|
||||
- 5678:5678
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
MASTER_EXEC_THREADS: "100"
|
||||
MASTER_EXEC_TASK_NUM: "20"
|
||||
MASTER_HEARTBEAT_INTERVAL: "10"
|
||||
MASTER_TASK_COMMIT_RETRYTIMES: "5"
|
||||
MASTER_TASK_COMMIT_INTERVAL: "1000"
|
||||
MASTER_MAX_CPULOAD_AVG: "100"
|
||||
MASTER_RESERVED_MEMORY: "0.1"
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
|
||||
interval: 30s
|
||||
|
|
@ -146,6 +102,7 @@ services:
|
|||
retries: 3
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
deploy:
|
||||
|
|
@ -153,43 +110,11 @@ services:
|
|||
replicas: 1
|
||||
|
||||
dolphinscheduler-worker:
|
||||
image: apache/dolphinscheduler:latest
|
||||
image: apache/dolphinscheduler:1.3.6-SNAPSHOT
|
||||
command: worker-server
|
||||
ports:
|
||||
- 1234:1234
|
||||
- 50051:50051
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
WORKER_EXEC_THREADS: "100"
|
||||
WORKER_HEARTBEAT_INTERVAL: "10"
|
||||
WORKER_MAX_CPULOAD_AVG: "100"
|
||||
WORKER_RESERVED_MEMORY: "0.1"
|
||||
WORKER_GROUPS: "default"
|
||||
WORKER_HOST_WEIGHT: "100"
|
||||
ALERT_LISTEN_HOST: dolphinscheduler-alert
|
||||
HADOOP_HOME: "/opt/soft/hadoop"
|
||||
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
|
||||
SPARK_HOME1: "/opt/soft/spark1"
|
||||
SPARK_HOME2: "/opt/soft/spark2"
|
||||
#PYTHON_HOME: "/opt/soft/python"
|
||||
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
|
||||
HIVE_HOME: "/opt/soft/hive"
|
||||
FLINK_HOME: "/opt/soft/flink"
|
||||
DATAX_HOME: "/opt/soft/datax"
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
|
||||
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
DATABASE_TYPE: postgresql
|
||||
DATABASE_DRIVER: org.postgresql.Driver
|
||||
DATABASE_HOST: dolphinscheduler-postgresql
|
||||
DATABASE_PORT: 5432
|
||||
DATABASE_USERNAME: root
|
||||
DATABASE_PASSWORD: root
|
||||
DATABASE_DATABASE: dolphinscheduler
|
||||
DATABASE_PARAMS: characterEncoding=utf8
|
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
|
||||
RESOURCE_STORAGE_TYPE: HDFS
|
||||
RESOURCE_UPLOAD_PATH: /dolphinscheduler
|
||||
FS_DEFAULT_FS: file:///
|
||||
env_file: config.env.sh
|
||||
healthcheck:
|
||||
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
|
||||
interval: 30s
|
||||
|
|
@ -198,6 +123,8 @@ services:
|
|||
volumes:
|
||||
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
deploy:
|
||||
|
|
@ -212,4 +139,6 @@ volumes:
|
|||
dolphinscheduler-postgresql:
|
||||
dolphinscheduler-zookeeper:
|
||||
dolphinscheduler-worker-data:
|
||||
dolphinscheduler-logs:
|
||||
dolphinscheduler-logs:
|
||||
dolphinscheduler-shared-local:
|
||||
dolphinscheduler-resource-local:
|
||||
|
|
@ -35,18 +35,18 @@ type: application
|
|||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
version: 1.2.0
|
||||
version: 2.0.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application.
|
||||
appVersion: 1.4.0
|
||||
appVersion: 1.3.6-SNAPSHOT
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: 10.x.x
|
||||
version: 10.3.18
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
- name: zookeeper
|
||||
version: 6.x.x
|
||||
version: 6.5.3
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: zookeeper.enabled
|
||||
|
|
|
|||
|
|
@ -1,364 +0,0 @@
|
|||
# DolphinScheduler
|
||||
|
||||
[DolphinScheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.
|
||||
|
||||
## Introduction
|
||||
This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Helm](https://helm.sh/) 3.1.0+
|
||||
- [Kubernetes](https://kubernetes.io/) 1.12+
|
||||
- PV provisioner support in the underlying infrastructure
|
||||
|
||||
## Installing the Chart
|
||||
|
||||
To install the chart with the release name `dolphinscheduler`:
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/apache/dolphinscheduler.git
|
||||
$ cd dolphinscheduler/docker/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler .
|
||||
```
|
||||
|
||||
To install the chart with a namespace named `test`:
|
||||
|
||||
```bash
|
||||
$ helm install dolphinscheduler . -n test
|
||||
```
|
||||
|
||||
> **Tip**: If a namespace named `test` is used, the option `-n test` needs to be added to the `helm` and `kubectl` command
|
||||
|
||||
These commands deploy DolphinScheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
|
||||
|
||||
> **Tip**: List all releases using `helm list`
|
||||
|
||||
## Access DolphinScheduler UI
|
||||
|
||||
If `ingress.enabled` in `values.yaml` is set to `true`, you just access `http://${ingress.host}/dolphinscheduler` in browser.
|
||||
|
||||
> **Tip**: If there is a problem with ingress access, please contact the Kubernetes administrator and refer to the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/)
|
||||
|
||||
Otherwise, you need to execute port-forward command like:
|
||||
|
||||
```bash
|
||||
$ kubectl port-forward --address 0.0.0.0 svc/dolphinscheduler-api 12345:12345
|
||||
$ kubectl port-forward --address 0.0.0.0 -n test svc/dolphinscheduler-api 12345:12345 # with test namespace
|
||||
```
|
||||
|
||||
> **Tip**: If the error of `unable to do port forwarding: socat not found` appears, you need to install `socat` at first
|
||||
|
||||
And then access the web: http://192.168.xx.xx:12345/dolphinscheduler
|
||||
|
||||
The default username is `admin` and the default password is `dolphinscheduler123`
|
||||
|
||||
> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds`
|
||||
|
||||
## Uninstalling the Chart
|
||||
|
||||
To uninstall/delete the `dolphinscheduler` deployment:
|
||||
|
||||
```bash
|
||||
$ helm uninstall dolphinscheduler
|
||||
```
|
||||
|
||||
The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release.
|
||||
|
||||
To delete the PVC's associated with `dolphinscheduler`:
|
||||
|
||||
```bash
|
||||
$ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler
|
||||
```
|
||||
|
||||
> **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it.
|
||||
|
||||
## Configuration
|
||||
|
||||
The Configuration file is `values.yaml`, and the following tables lists the configurable parameters of the DolphinScheduler chart and their default values.
|
||||
|
||||
| Parameter | Description | Default |
|
||||
| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- |
|
||||
| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` |
|
||||
| | | |
|
||||
| `image.repository` | Docker image repository for the DolphinScheduler | `apache/dolphinscheduler` |
|
||||
| `image.tag` | Docker image version for the DolphinScheduler | `latest` |
|
||||
| `image.pullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` |
|
||||
| `image.pullSecret` | Image pull secret. An optional reference to secret in the same namespace to use for pulling any of the images | `nil` |
|
||||
| | | |
|
||||
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL | `true` |
|
||||
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
|
||||
| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` |
|
||||
| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` |
|
||||
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
|
||||
| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` |
|
||||
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` |
|
||||
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |
|
||||
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database port will use it | `5432` |
|
||||
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database username will use it | `root` |
|
||||
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database password will use it | `root` |
|
||||
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database database will use it | `dolphinscheduler` |
|
||||
| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database params will use it | `characterEncoding=utf8` |
|
||||
| | | |
|
||||
| `zookeeper.enabled` | If not exists external Zookeeper, by default, the DolphinScheduler will use a internal Zookeeper | `true` |
|
||||
| `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` |
|
||||
| `zookeeper.service.port` | ZooKeeper port | `2181` |
|
||||
| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` |
|
||||
| `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `zookeeper.zookeeperRoot` | Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` |
|
||||
| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
|
||||
| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` |
|
||||
| | | |
|
||||
| `common.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` |
|
||||
| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` |
|
||||
| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` |
|
||||
| `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` |
|
||||
| `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` |
|
||||
| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `s3.xxx.amazonaws.com` |
|
||||
| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
|
||||
| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
|
||||
| `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` |
|
||||
| `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
|
||||
| `common.fsFileResourcePersistence.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` |
|
||||
| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| | | |
|
||||
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
|
||||
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
|
||||
| `master.annotations` | The `annotations` for master server | `{}` |
|
||||
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `master.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `master.resources` | The `resource` limit and request config for master server | `{}` |
|
||||
| `master.configmap.DOLPHINSCHEDULER_OPTS` | The java options for master server | `""` |
|
||||
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread number | `100` |
|
||||
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` |
|
||||
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` |
|
||||
| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` |
|
||||
| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` |
|
||||
| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` |
|
||||
| `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` |
|
||||
| `master.configmap.MASTER_LISTEN_PORT` | Master listen port | `5678` |
|
||||
| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` |
|
||||
| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
|
||||
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| | | |
|
||||
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
|
||||
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
|
||||
| `worker.annotations` | The `annotations` for worker server | `{}` |
|
||||
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `worker.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `worker.resources` | The `resource` limit and request config for worker server | `{}` |
|
||||
| `worker.configmap.DOLPHINSCHEDULER_OPTS` | The java options for worker server | `""` |
|
||||
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread number | `100` |
|
||||
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` |
|
||||
| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` |
|
||||
| `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` |
|
||||
| `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` |
|
||||
| `worker.configmap.WORKER_GROUPS` | Worker groups | `default` |
|
||||
| `worker.configmap.WORKER_HOST_WEIGHT` | Worker host weight | `100` |
|
||||
| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| | | |
|
||||
| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
|
||||
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
|
||||
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
|
||||
| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
|
||||
| `alert.annotations` | The `annotations` for alert server | `{}` |
|
||||
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `alert.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `alert.resources` | The `resource` limit and request config for alert server | `{}` |
|
||||
| `alert.configmap.DOLPHINSCHEDULER_OPTS` | The java options for alert server | `""` |
|
||||
| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin directory | `lib/plugin/alert` |
|
||||
| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` |
|
||||
| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
|
||||
| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| | | |
|
||||
| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
|
||||
| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
|
||||
| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
|
||||
| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
|
||||
| `api.annotations` | The `annotations` for api server | `{}` |
|
||||
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `api.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `api.resources` | The `resource` limit and request config for api server | `{}` |
|
||||
| `api.configmap.DOLPHINSCHEDULER_OPTS` | The java options for api server | `""` |
|
||||
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` |
|
||||
| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
|
||||
| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
|
||||
| | | |
|
||||
| `ingress.enabled` | Enable ingress | `false` |
|
||||
| `ingress.host` | Ingress host | `dolphinscheduler.org` |
|
||||
| `ingress.path` | Ingress path | `/dolphinscheduler` |
|
||||
| `ingress.tls.enabled` | Enable ingress tls | `false` |
|
||||
| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` |
|
||||
|
||||
## FAQ
|
||||
|
||||
### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver and client of MySQL.
|
||||
>
|
||||
> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver and client:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
RUN apk add --update --no-cache mysql-client
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver and client:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:mysql` to a docker registry
|
||||
|
||||
5. Modify image `repository` and update `tag` to `mysql` in `values.yaml`
|
||||
|
||||
6. Modify postgresql `enabled` to `false`
|
||||
|
||||
7. Modify externalDatabase (especially modify `host`, `username` and `password`):
|
||||
|
||||
```
|
||||
externalDatabase:
|
||||
type: "mysql"
|
||||
driver: "com.mysql.jdbc.Driver"
|
||||
host: "localhost"
|
||||
port: "3306"
|
||||
username: "root"
|
||||
password: "root"
|
||||
database: "dolphinscheduler"
|
||||
params: "useUnicode=true&characterEncoding=UTF-8"
|
||||
```
|
||||
|
||||
8. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
|
||||
|
||||
### How to support MySQL datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of MySQL.
|
||||
>
|
||||
> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql-driver .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry
|
||||
|
||||
5. Modify image `repository` and update `tag` to `mysql-driver` in `values.yaml`
|
||||
|
||||
6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
|
||||
|
||||
7. Add a MySQL datasource in `Datasource manage`
|
||||
|
||||
### How to support Oracle datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of Oracle.
|
||||
>
|
||||
> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
|
||||
|
||||
1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
|
||||
|
||||
2. Create a new `Dockerfile` to add Oracle driver:
|
||||
|
||||
```
|
||||
FROM apache/dolphinscheduler:latest
|
||||
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including Oracle driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:oracle-driver .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:oracle-driver` to a docker registry
|
||||
|
||||
5. Modify image `repository` and update `tag` to `oracle-driver` in `values.yaml`
|
||||
|
||||
6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
|
||||
|
||||
7. Add a Oracle datasource in `Datasource manage`
|
||||
|
||||
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.
|
||||
|
|
@ -17,16 +17,34 @@
|
|||
|
||||
** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **
|
||||
|
||||
Access DolphinScheduler by:
|
||||
Access DolphinScheduler UI URL by:
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
|
||||
DolphinScheduler URL: http://{{ .Values.ingress.host }}/dolphinscheduler
|
||||
DolphinScheduler UI URL: http{{ if .Values.ingress.tls.enabled }}s{{ end }}://{{ .Values.ingress.host }}/dolphinscheduler
|
||||
|
||||
{{- else }}
|
||||
{{- else if eq .Values.api.service.type "ClusterIP" }}
|
||||
|
||||
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345
|
||||
kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345
|
||||
|
||||
DolphinScheduler URL: http://127.0.0.1:12345/dolphinscheduler
|
||||
DolphinScheduler UI URL: http://127.0.0.1:12345/dolphinscheduler
|
||||
|
||||
{{- else if eq .Values.api.service.type "NodePort" }}
|
||||
|
||||
NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
NODE_PORT=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
|
||||
echo http://$NODE_IP:$NODE_PORT/dolphinscheduler
|
||||
|
||||
DolphinScheduler UI URL: http://$NODE_IP:$NODE_PORT/dolphinscheduler
|
||||
|
||||
{{- else if eq .Values.api.service.type "LoadBalancer" }}
|
||||
|
||||
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
|
||||
You can watch the status of by running 'kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -w'
|
||||
|
||||
SERVICE_IP=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
|
||||
echo http://$SERVICE_IP:12345/dolphinscheduler
|
||||
|
||||
DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler
|
||||
|
||||
{{- end }}
|
||||
|
|
@ -30,7 +30,52 @@ If release name contains chart name it will be used as a full name.
|
|||
Create a default docker image fullname.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.image.fullname" -}}
|
||||
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
|
||||
{{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default common labels.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.common.labels" -}}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a master labels.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.master.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/component: master
|
||||
{{ include "dolphinscheduler.common.labels" . }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a worker labels.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.worker.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/component: worker
|
||||
{{ include "dolphinscheduler.common.labels" . }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create an alert labels.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.alert.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/component: alert
|
||||
{{ include "dolphinscheduler.common.labels" . }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create an api labels.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.api.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/component: api
|
||||
{{ include "dolphinscheduler.common.labels" . }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
|
|
@ -57,4 +102,135 @@ Create a default fully qualified zookkeeper quorum.
|
|||
{{- define "dolphinscheduler.zookeeper.quorum" -}}
|
||||
{{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}}
|
||||
{{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a database environment variables.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.database.env_vars" -}}
|
||||
- name: DATABASE_TYPE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "postgresql"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.type | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_DRIVER
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "org.postgresql.Driver"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.driver | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_HOST
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.host | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PORT
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "5432"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.port | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_USERNAME
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlUsername }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.username | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
key: postgresql-password
|
||||
{{- else }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
key: database-password
|
||||
{{- end }}
|
||||
- name: DATABASE_DATABASE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlDatabase }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.database | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PARAMS
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "characterEncoding=utf8"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.params | quote }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a zookeeper environment variables.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.zookeeper.env_vars" -}}
|
||||
- name: ZOOKEEPER_QUORUM
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_ROOT
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ .Values.zookeeper.zookeeperRoot }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperRoot }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a common fs_s3a environment variables.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.fs_s3a.env_vars" -}}
|
||||
{{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" -}}
|
||||
- name: FS_S3A_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
key: fs-s3a-secret-key
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a sharedStoragePersistence volume.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.sharedStorage.volume" -}}
|
||||
{{- if .Values.common.sharedStoragePersistence.enabled -}}
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-shared
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "dolphinscheduler.fullname" . }}-shared
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a sharedStoragePersistence volumeMount.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.sharedStorage.volumeMount" -}}
|
||||
{{- if .Values.common.sharedStoragePersistence.enabled -}}
|
||||
- mountPath: {{ .Values.common.sharedStoragePersistence.mountPath | quote }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-shared
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a fsFileResourcePersistence volume.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.fsFileResource.volume" -}}
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled -}}
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a fsFileResourcePersistence volumeMount.
|
||||
*/}}
|
||||
{{- define "dolphinscheduler.fsFileResource.volumeMount" -}}
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled -}}
|
||||
- mountPath: {{ default "/dolphinscheduler" .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
|
@ -21,9 +21,9 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
data:
|
||||
DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }}
|
||||
ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }}
|
||||
{{- range $key, $value := .Values.alert.configmap }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -21,8 +21,9 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
data:
|
||||
DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }}
|
||||
{{- range $key, $value := .Values.api.configmap }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -21,17 +21,9 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
data:
|
||||
DOLPHINSCHEDULER_ENV: |-
|
||||
{{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }}
|
||||
{{ . }}
|
||||
{{- range $key, $value := (omit .Values.common.configmap "FS_S3A_SECRET_KEY") }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
|
||||
RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
|
||||
RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
|
||||
FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
|
||||
FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
|
||||
FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -21,16 +21,9 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
data:
|
||||
DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }}
|
||||
MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }}
|
||||
MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }}
|
||||
MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }}
|
||||
MASTER_TASK_COMMIT_RETRYTIMES: {{ .Values.master.configmap.MASTER_TASK_COMMIT_RETRYTIMES | quote }}
|
||||
MASTER_TASK_COMMIT_INTERVAL: {{ .Values.master.configmap.MASTER_TASK_COMMIT_INTERVAL | quote }}
|
||||
MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }}
|
||||
MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }}
|
||||
MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }}
|
||||
{{- range $key, $value := .Values.master.configmap }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -21,15 +21,9 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
data:
|
||||
DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }}
|
||||
WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }}
|
||||
WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }}
|
||||
WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }}
|
||||
WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }}
|
||||
WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }}
|
||||
WORKER_GROUPS: {{ .Values.worker.configmap.WORKER_GROUPS | quote }}
|
||||
WORKER_HOST_WEIGHT: {{ .Values.worker.configmap.WORKER_HOST_WEIGHT | quote }}
|
||||
{{- range $key, $value := .Values.worker.configmap }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -19,18 +19,12 @@ kind: Deployment
|
|||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: alert
|
||||
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.alert.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: alert
|
||||
{{- include "dolphinscheduler.alert.labels" . | nindent 6 }}
|
||||
strategy:
|
||||
type: {{ .Values.alert.strategy.type | quote }}
|
||||
rollingUpdate:
|
||||
|
|
@ -39,23 +33,23 @@ spec:
|
|||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: alert
|
||||
{{- include "dolphinscheduler.alert.labels" . | nindent 8 }}
|
||||
{{- if .Values.alert.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.alert.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.alert.affinity }}
|
||||
affinity: {{- toYaml .Values.alert.affinity | nindent 8 }}
|
||||
affinity:
|
||||
{{- toYaml .Values.alert.affinity | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.alert.nodeSelector }}
|
||||
nodeSelector: {{- toYaml .Values.alert.nodeSelector | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- toYaml .Values.alert.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.alert.tolerations }}
|
||||
tolerations: {{- toYaml . | nindent 8 }}
|
||||
tolerations:
|
||||
{{- toYaml .Values.alert.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.image.pullSecret }}
|
||||
imagePullSecrets:
|
||||
|
|
@ -73,84 +67,20 @@ spec:
|
|||
env:
|
||||
- name: TZ
|
||||
value: {{ .Values.timezone }}
|
||||
- name: DOLPHINSCHEDULER_OPTS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: DOLPHINSCHEDULER_OPTS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
- name: ALERT_PLUGIN_DIR
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: ALERT_PLUGIN_DIR
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
- name: DATABASE_TYPE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "postgresql"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.type | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_DRIVER
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "org.postgresql.Driver"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.driver | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_HOST
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.host | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PORT
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "5432"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.port | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_USERNAME
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlUsername }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.username | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
key: postgresql-password
|
||||
{{- else }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
key: database-password
|
||||
{{- end }}
|
||||
- name: DATABASE_DATABASE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlDatabase }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.database | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PARAMS
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "characterEncoding=utf8"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.params | quote }}
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
{{- if .Values.alert.resources }}
|
||||
resources:
|
||||
limits:
|
||||
memory: {{ .Values.alert.resources.limits.memory }}
|
||||
cpu: {{ .Values.alert.resources.limits.cpu }}
|
||||
requests:
|
||||
memory: {{ .Values.alert.resources.requests.memory }}
|
||||
cpu: {{ .Values.alert.resources.requests.cpu }}
|
||||
{{- toYaml .Values.alert.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.alert.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- AlertServer
|
||||
command: ["bash", "/root/checkpoint.sh", "AlertServer"]
|
||||
initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }}
|
||||
|
|
@ -160,10 +90,7 @@ spec:
|
|||
{{- if .Values.alert.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- AlertServer
|
||||
command: ["bash", "/root/checkpoint.sh", "AlertServer"]
|
||||
initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }}
|
||||
|
|
|
|||
|
|
@ -19,18 +19,12 @@ kind: Deployment
|
|||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- include "dolphinscheduler.api.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.api.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- include "dolphinscheduler.api.labels" . | nindent 6 }}
|
||||
strategy:
|
||||
type: {{ .Values.api.strategy.type | quote }}
|
||||
rollingUpdate:
|
||||
|
|
@ -39,23 +33,23 @@ spec:
|
|||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- if .Values.alert.annotations }}
|
||||
{{- include "dolphinscheduler.api.labels" . | nindent 8 }}
|
||||
{{- if .Values.api.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.alert.annotations | nindent 8 }}
|
||||
{{- toYaml .Values.api.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.api.affinity }}
|
||||
affinity: {{- toYaml .Values.api.affinity | nindent 8 }}
|
||||
affinity:
|
||||
{{- toYaml .Values.api.affinity | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.api.nodeSelector }}
|
||||
nodeSelector: {{- toYaml .Values.api.nodeSelector | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- toYaml .Values.api.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.api.tolerations }}
|
||||
tolerations: {{- toYaml . | nindent 8 }}
|
||||
tolerations:
|
||||
{{- toYaml .Values.api.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.image.pullSecret }}
|
||||
imagePullSecrets:
|
||||
|
|
@ -73,120 +67,22 @@ spec:
|
|||
env:
|
||||
- name: TZ
|
||||
value: {{ .Values.timezone }}
|
||||
- name: DOLPHINSCHEDULER_OPTS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: DOLPHINSCHEDULER_OPTS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
- name: DATABASE_TYPE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "postgresql"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.type | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_DRIVER
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "org.postgresql.Driver"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.driver | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_HOST
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.host | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PORT
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "5432"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.port | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_USERNAME
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlUsername }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.username | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
key: postgresql-password
|
||||
{{- else }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
key: database-password
|
||||
{{- end }}
|
||||
- name: DATABASE_DATABASE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlDatabase }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.database | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PARAMS
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "characterEncoding=utf8"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.params | quote }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_QUORUM
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_ROOT
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ .Values.zookeeper.zookeeperRoot }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperRoot }}
|
||||
{{- end }}
|
||||
- name: RESOURCE_STORAGE_TYPE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: RESOURCE_STORAGE_TYPE
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: RESOURCE_UPLOAD_PATH
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: RESOURCE_UPLOAD_PATH
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_DEFAULT_FS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_DEFAULT_FS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
|
||||
- name: FS_S3A_ENDPOINT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_S3A_ENDPOINT
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_S3A_ACCESS_KEY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_S3A_ACCESS_KEY
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_S3A_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
key: fs-s3a-secret-key
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
{{- if .Values.api.resources }}
|
||||
resources:
|
||||
limits:
|
||||
memory: {{ .Values.api.resources.limits.memory | quote }}
|
||||
cpu: {{ .Values.api.resources.limits.cpu | quote }}
|
||||
requests:
|
||||
memory: {{ .Values.api.resources.requests.memory | quote }}
|
||||
cpu: {{ .Values.api.resources.requests.cpu | quote }}
|
||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.api.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 12345
|
||||
exec:
|
||||
command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"]
|
||||
initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }}
|
||||
|
|
@ -195,8 +91,8 @@ spec:
|
|||
{{- end }}
|
||||
{{- if .Values.api.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 12345
|
||||
exec:
|
||||
command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"]
|
||||
initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }}
|
||||
|
|
@ -206,10 +102,8 @@ spec:
|
|||
volumeMounts:
|
||||
- mountPath: "/opt/dolphinscheduler/logs"
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled }}
|
||||
- mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
|
||||
volumes:
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
{{- if .Values.api.persistentVolumeClaim.enabled }}
|
||||
|
|
@ -218,8 +112,5 @@ spec:
|
|||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled }}
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
|
||||
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
|
||||
|
|
@ -27,8 +27,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
rules:
|
||||
- host: {{ .Values.ingress.host }}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.alert.persistentVolumeClaim.accessModes }}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.api.persistentVolumeClaim.accessModes }}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/resource-policy": keep
|
||||
spec:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
{{- if .Values.common.sharedStoragePersistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-shared
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-shared
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/resource-policy": keep
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.common.sharedStoragePersistence.accessModes }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
storageClassName: {{ .Values.common.sharedStoragePersistence.storageClassName | quote }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.common.sharedStoragePersistence.storage | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -21,8 +21,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
database-password: {{ .Values.externalDatabase.password | b64enc | quote }}
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
|
||||
{{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }}
|
||||
|
|
|
|||
|
|
@ -19,40 +19,34 @@ kind: StatefulSet
|
|||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- include "dolphinscheduler.master.labels" . | nindent 4 }}
|
||||
spec:
|
||||
podManagementPolicy: {{ .Values.master.podManagementPolicy }}
|
||||
replicas: {{ .Values.master.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- include "dolphinscheduler.master.labels" . | nindent 6 }}
|
||||
serviceName: {{ template "dolphinscheduler.fullname" . }}-master-headless
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- if .Values.alert.annotations }}
|
||||
{{- include "dolphinscheduler.master.labels" . | nindent 8 }}
|
||||
{{- if .Values.master.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.alert.annotations | nindent 8 }}
|
||||
{{- toYaml .Values.master.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.master.affinity }}
|
||||
affinity: {{- toYaml .Values.master.affinity | nindent 8 }}
|
||||
affinity:
|
||||
{{- toYaml .Values.master.affinity | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.nodeSelector }}
|
||||
nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- toYaml .Values.master.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.tolerations }}
|
||||
tolerations: {{- toYaml . | nindent 8 }}
|
||||
tolerations:
|
||||
{{- toYaml .Values.master.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.image.pullSecret }}
|
||||
imagePullSecrets:
|
||||
|
|
@ -65,141 +59,27 @@ spec:
|
|||
args:
|
||||
- "master-server"
|
||||
ports:
|
||||
- containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }}
|
||||
- containerPort: 5678
|
||||
name: "master-port"
|
||||
env:
|
||||
- name: TZ
|
||||
value: {{ .Values.timezone }}
|
||||
- name: DOLPHINSCHEDULER_OPTS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: DOLPHINSCHEDULER_OPTS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
- name: MASTER_EXEC_THREADS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_EXEC_THREADS
|
||||
- name: MASTER_EXEC_TASK_NUM
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_EXEC_TASK_NUM
|
||||
- name: MASTER_HEARTBEAT_INTERVAL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_HEARTBEAT_INTERVAL
|
||||
- name: MASTER_TASK_COMMIT_RETRYTIMES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_TASK_COMMIT_RETRYTIMES
|
||||
- name: MASTER_TASK_COMMIT_INTERVAL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_TASK_COMMIT_INTERVAL
|
||||
- name: MASTER_MAX_CPULOAD_AVG
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_MAX_CPULOAD_AVG
|
||||
- name: MASTER_RESERVED_MEMORY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_RESERVED_MEMORY
|
||||
- name: MASTER_LISTEN_PORT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
key: MASTER_LISTEN_PORT
|
||||
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
|
||||
- name: DATABASE_TYPE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "postgresql"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.type | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_DRIVER
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "org.postgresql.Driver"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.driver | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_HOST
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.host | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PORT
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "5432"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.port | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_USERNAME
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlUsername }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.username | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
key: postgresql-password
|
||||
{{- else }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
key: database-password
|
||||
{{- end }}
|
||||
- name: DATABASE_DATABASE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlDatabase }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.database | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PARAMS
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "characterEncoding=utf8"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.params | quote }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_QUORUM
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_ROOT
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ .Values.zookeeper.zookeeperRoot }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperRoot }}
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
{{- if .Values.master.resources }}
|
||||
resources:
|
||||
limits:
|
||||
memory: {{ .Values.master.resources.limits.memory | quote }}
|
||||
cpu: {{ .Values.master.resources.limits.cpu | quote }}
|
||||
requests:
|
||||
memory: {{ .Values.master.resources.requests.memory | quote }}
|
||||
cpu: {{ .Values.master.resources.requests.cpu | quote }}
|
||||
{{- toYaml .Values.master.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- MasterServer
|
||||
command: ["bash", "/root/checkpoint.sh", "MasterServer"]
|
||||
initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }}
|
||||
|
|
@ -209,10 +89,7 @@ spec:
|
|||
{{- if .Values.master.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- MasterServer
|
||||
command: ["bash", "/root/checkpoint.sh", "MasterServer"]
|
||||
initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }}
|
||||
|
|
@ -222,6 +99,7 @@ spec:
|
|||
volumeMounts:
|
||||
- mountPath: "/opt/dolphinscheduler/logs"
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
|
||||
volumes:
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
{{- if .Values.master.persistentVolumeClaim.enabled }}
|
||||
|
|
@ -230,19 +108,19 @@ spec:
|
|||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
|
||||
{{- if .Values.master.persistentVolumeClaim.enabled }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 10 }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.master.persistentVolumeClaim.accessModes }}
|
||||
{{- range .Values.master.persistentVolumeClaim.accessModes }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }}
|
||||
resources:
|
||||
requests:
|
||||
|
|
|
|||
|
|
@ -19,40 +19,34 @@ kind: StatefulSet
|
|||
metadata:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
|
||||
spec:
|
||||
podManagementPolicy: {{ .Values.worker.podManagementPolicy }}
|
||||
replicas: {{ .Values.worker.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- include "dolphinscheduler.worker.labels" . | nindent 6 }}
|
||||
serviceName: {{ template "dolphinscheduler.fullname" . }}-worker-headless
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- if .Values.alert.annotations }}
|
||||
{{- include "dolphinscheduler.worker.labels" . | nindent 8 }}
|
||||
{{- if .Values.worker.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.alert.annotations | nindent 8 }}
|
||||
{{- toYaml .Values.worker.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.worker.affinity }}
|
||||
affinity: {{- toYaml .Values.worker.affinity | nindent 8 }}
|
||||
affinity:
|
||||
{{- toYaml .Values.worker.affinity | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.nodeSelector }}
|
||||
nodeSelector: {{- toYaml .Values.worker.nodeSelector | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- toYaml .Values.worker.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.tolerations }}
|
||||
tolerations: {{- toYaml . | nindent 8 }}
|
||||
tolerations:
|
||||
{{- toYaml .Values.worker.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.image.pullSecret }}
|
||||
imagePullSecrets:
|
||||
|
|
@ -65,172 +59,33 @@ spec:
|
|||
args:
|
||||
- "worker-server"
|
||||
ports:
|
||||
- containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }}
|
||||
- containerPort: 1234
|
||||
name: "worker-port"
|
||||
- containerPort: 50051
|
||||
name: "logger-port"
|
||||
env:
|
||||
- name: TZ
|
||||
value: {{ .Values.timezone }}
|
||||
- name: DOLPHINSCHEDULER_OPTS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: DOLPHINSCHEDULER_OPTS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
- name: WORKER_EXEC_THREADS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_EXEC_THREADS
|
||||
- name: WORKER_HEARTBEAT_INTERVAL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_HEARTBEAT_INTERVAL
|
||||
- name: WORKER_MAX_CPULOAD_AVG
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_MAX_CPULOAD_AVG
|
||||
- name: WORKER_RESERVED_MEMORY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_RESERVED_MEMORY
|
||||
- name: WORKER_LISTEN_PORT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_LISTEN_PORT
|
||||
- name: WORKER_GROUPS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_GROUPS
|
||||
- name: WORKER_WEUGHT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
key: WORKER_HOST_WEIGHT
|
||||
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
|
||||
- name: ALERT_LISTEN_HOST
|
||||
value: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
- name: DATABASE_TYPE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "postgresql"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.type | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_DRIVER
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "org.postgresql.Driver"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.driver | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_HOST
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.host | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PORT
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "5432"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.port | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_USERNAME
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlUsername }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.username | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
|
||||
key: postgresql-password
|
||||
{{- else }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
|
||||
key: database-password
|
||||
{{- end }}
|
||||
- name: DATABASE_DATABASE
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: {{ .Values.postgresql.postgresqlDatabase }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.database | quote }}
|
||||
{{- end }}
|
||||
- name: DATABASE_PARAMS
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
value: "characterEncoding=utf8"
|
||||
{{- else }}
|
||||
value: {{ .Values.externalDatabase.params | quote }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_QUORUM
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
|
||||
{{- end }}
|
||||
- name: ZOOKEEPER_ROOT
|
||||
{{- if .Values.zookeeper.enabled }}
|
||||
value: {{ .Values.zookeeper.zookeeperRoot }}
|
||||
{{- else }}
|
||||
value: {{ .Values.externalZookeeper.zookeeperRoot }}
|
||||
{{- end }}
|
||||
- name: RESOURCE_STORAGE_TYPE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: RESOURCE_STORAGE_TYPE
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: RESOURCE_UPLOAD_PATH
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: RESOURCE_UPLOAD_PATH
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_DEFAULT_FS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_DEFAULT_FS
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
|
||||
- name: FS_S3A_ENDPOINT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_S3A_ENDPOINT
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_S3A_ACCESS_KEY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: FS_S3A_ACCESS_KEY
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- name: FS_S3A_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
key: fs-s3a-secret-key
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
- configMapRef:
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
{{- if .Values.worker.resources }}
|
||||
resources:
|
||||
limits:
|
||||
memory: {{ .Values.worker.resources.limits.memory | quote }}
|
||||
cpu: {{ .Values.worker.resources.limits.cpu | quote }}
|
||||
requests:
|
||||
memory: {{ .Values.worker.resources.requests.memory | quote }}
|
||||
cpu: {{ .Values.worker.resources.requests.cpu | quote }}
|
||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- WorkerServer
|
||||
command: ["bash", "/root/checkpoint.sh", "WorkerServer"]
|
||||
initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
|
||||
|
|
@ -240,10 +95,7 @@ spec:
|
|||
{{- if .Values.worker.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- /root/checkpoint.sh
|
||||
- WorkerServer
|
||||
command: ["bash", "/root/checkpoint.sh", "WorkerServer"]
|
||||
initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
|
||||
|
|
@ -251,17 +103,12 @@ spec:
|
|||
failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- mountPath: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
|
||||
- mountPath: {{ default "/tmp/dolphinscheduler" .Values.common.configmap.DATA_BASEDIR_PATH | quote }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker-data
|
||||
- mountPath: "/opt/dolphinscheduler/logs"
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
|
||||
- mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"
|
||||
subPath: "dolphinscheduler_env.sh"
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common-env
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled }}
|
||||
- mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
|
||||
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
|
||||
volumes:
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data
|
||||
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
|
||||
|
|
@ -277,18 +124,8 @@ spec:
|
|||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-common-env
|
||||
configMap:
|
||||
defaultMode: 0777
|
||||
name: {{ include "dolphinscheduler.fullname" . }}-common
|
||||
items:
|
||||
- key: DOLPHINSCHEDULER_ENV
|
||||
path: dolphinscheduler_env.sh
|
||||
{{- if .Values.common.fsFileResourcePersistence.enabled }}
|
||||
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
|
||||
{{- end }}
|
||||
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
|
||||
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
|
||||
{{- if .Values.worker.persistentVolumeClaim.enabled }}
|
||||
volumeClaimTemplates:
|
||||
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
|
||||
|
|
@ -296,8 +133,7 @@ spec:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-worker-data
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 10 }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }}
|
||||
|
|
@ -313,8 +149,7 @@ spec:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 10 }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
ports:
|
||||
- port: 50052
|
||||
|
|
@ -29,7 +28,4 @@ spec:
|
|||
protocol: TCP
|
||||
name: alert-port
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: alert
|
||||
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
|
||||
|
|
@ -20,16 +20,35 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
{{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }}
|
||||
annotations:
|
||||
{{- range $key, $value := .Values.api.service.annotations }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.api.service.type }}
|
||||
{{- if and (eq .Values.api.service.type "ClusterIP") .Values.api.service.clusterIP }}
|
||||
clusterIP: {{ .Values.api.service.clusterIP }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- port: 12345
|
||||
targetPort: api-port
|
||||
{{- if and (eq .Values.api.service.type "NodePort") .Values.api.service.nodePort }}
|
||||
nodePort: {{ .Values.api.service.nodePort }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
name: api-port
|
||||
{{- if .Values.api.service.externalIPs }}
|
||||
externalIPs:
|
||||
{{- toYaml .Values.api.service.externalIPs | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if and (eq .Values.api.service.type "ExternalName") .Values.api.service.externalName }}
|
||||
externalName: {{ .Values.api.service.externalName }}
|
||||
{{- end }}
|
||||
{{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.loadBalancerIP }}
|
||||
loadBalancerIP: {{ .Values.api.service.loadBalancerIP }}
|
||||
{{- end }}
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- include "dolphinscheduler.api.labels" . | nindent 4 }}
|
||||
|
|
@ -20,17 +20,13 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-master-headless
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
clusterIP: "None"
|
||||
ports:
|
||||
- port: {{ .Values.master.configmap.MASTER_LISTEN_PORT }}
|
||||
- port: 5678
|
||||
targetPort: master-port
|
||||
protocol: TCP
|
||||
name: master-port
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- include "dolphinscheduler.master.labels" . | nindent 4 }}
|
||||
|
|
@ -20,12 +20,11 @@ metadata:
|
|||
name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
clusterIP: "None"
|
||||
ports:
|
||||
- port: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }}
|
||||
- port: 1234
|
||||
targetPort: worker-port
|
||||
protocol: TCP
|
||||
name: worker-port
|
||||
|
|
@ -34,7 +33,4 @@ spec:
|
|||
protocol: TCP
|
||||
name: logger-port
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
|
||||
|
|
@ -23,7 +23,7 @@ timezone: "Asia/Shanghai"
|
|||
|
||||
image:
|
||||
repository: "apache/dolphinscheduler"
|
||||
tag: "latest"
|
||||
tag: "1.3.6-SNAPSHOT"
|
||||
pullPolicy: "IfNotPresent"
|
||||
pullSecret: ""
|
||||
|
||||
|
|
@ -54,8 +54,6 @@ externalDatabase:
|
|||
zookeeper:
|
||||
enabled: true
|
||||
fourlwCommandsWhitelist: "srvr,ruok,wchs,cons"
|
||||
service:
|
||||
port: "2181"
|
||||
persistence:
|
||||
enabled: false
|
||||
size: "20Gi"
|
||||
|
|
@ -69,26 +67,48 @@ externalZookeeper:
|
|||
zookeeperRoot: "/dolphinscheduler"
|
||||
|
||||
common:
|
||||
## ConfigMap
|
||||
## Configmap
|
||||
configmap:
|
||||
DOLPHINSCHEDULER_ENV:
|
||||
- "export HADOOP_HOME=/opt/soft/hadoop"
|
||||
- "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop"
|
||||
- "export SPARK_HOME1=/opt/soft/spark1"
|
||||
- "export SPARK_HOME2=/opt/soft/spark2"
|
||||
#- "export PYTHON_HOME=/opt/soft/python"
|
||||
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
|
||||
- "export HIVE_HOME=/opt/soft/hive"
|
||||
- "export FLINK_HOME=/opt/soft/flink"
|
||||
- "export DATAX_HOME=/opt/soft/datax"
|
||||
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH"
|
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
|
||||
DOLPHINSCHEDULER_OPTS: ""
|
||||
DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
|
||||
RESOURCE_STORAGE_TYPE: "HDFS"
|
||||
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
|
||||
FS_DEFAULT_FS: "file:///"
|
||||
FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
|
||||
FS_S3A_ACCESS_KEY: "xxxxxxx"
|
||||
FS_S3A_SECRET_KEY: "xxxxxxx"
|
||||
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE: "false"
|
||||
JAVA_SECURITY_KRB5_CONF_PATH: "/opt/krb5.conf"
|
||||
LOGIN_USER_KEYTAB_USERNAME: "hdfs@HADOOP.COM"
|
||||
LOGIN_USER_KEYTAB_PATH: "/opt/hdfs.keytab"
|
||||
KERBEROS_EXPIRE_TIME: "2"
|
||||
HDFS_ROOT_USER: "hdfs"
|
||||
RESOURCE_MANAGER_HTTPADDRESS_PORT: "8088"
|
||||
YARN_RESOURCEMANAGER_HA_RM_IDS: ""
|
||||
YARN_APPLICATION_STATUS_ADDRESS: "http://ds1:%s/ws/v1/cluster/apps/%s"
|
||||
YARN_JOB_HISTORY_STATUS_ADDRESS: "http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"
|
||||
DATASOURCE_ENCRYPTION_ENABLE: "false"
|
||||
DATASOURCE_ENCRYPTION_SALT: "!@#$%^&*"
|
||||
SUDO_ENABLE: "true"
|
||||
# dolphinscheduler env
|
||||
HADOOP_HOME: "/opt/soft/hadoop"
|
||||
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
|
||||
SPARK_HOME1: "/opt/soft/spark1"
|
||||
SPARK_HOME2: "/opt/soft/spark2"
|
||||
PYTHON_HOME: "/usr/bin/python"
|
||||
JAVA_HOME: "/usr/local/openjdk-8"
|
||||
HIVE_HOME: "/opt/soft/hive"
|
||||
FLINK_HOME: "/opt/soft/flink"
|
||||
DATAX_HOME: "/opt/soft/datax"
|
||||
## Shared storage persistence mounted into api, master and worker, such as Hadoop, Spark, Flink and DataX binary package
|
||||
sharedStoragePersistence:
|
||||
enabled: false
|
||||
mountPath: "/opt/soft"
|
||||
accessModes:
|
||||
- "ReadWriteMany"
|
||||
## storageClassName must support the access mode: ReadWriteMany
|
||||
storageClassName: "-"
|
||||
storage: "20Gi"
|
||||
## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage
|
||||
fsFileResourcePersistence:
|
||||
enabled: false
|
||||
|
|
@ -126,17 +146,18 @@ master:
|
|||
# requests:
|
||||
# memory: "2Gi"
|
||||
# cpu: "500m"
|
||||
## ConfigMap
|
||||
## Configmap
|
||||
configmap:
|
||||
DOLPHINSCHEDULER_OPTS: ""
|
||||
MASTER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
MASTER_EXEC_THREADS: "100"
|
||||
MASTER_EXEC_TASK_NUM: "20"
|
||||
MASTER_DISPATCH_TASK_NUM: "3"
|
||||
MASTER_HOST_SELECTOR: "LowerWeight"
|
||||
MASTER_HEARTBEAT_INTERVAL: "10"
|
||||
MASTER_TASK_COMMIT_RETRYTIMES: "5"
|
||||
MASTER_TASK_COMMIT_INTERVAL: "1000"
|
||||
MASTER_MAX_CPULOAD_AVG: "100"
|
||||
MASTER_RESERVED_MEMORY: "0.1"
|
||||
MASTER_LISTEN_PORT: "5678"
|
||||
MASTER_MAX_CPULOAD_AVG: "-1"
|
||||
MASTER_RESERVED_MEMORY: "0.3"
|
||||
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
|
||||
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
livenessProbe:
|
||||
|
|
@ -194,16 +215,16 @@ worker:
|
|||
# requests:
|
||||
# memory: "2Gi"
|
||||
# cpu: "500m"
|
||||
## ConfigMap
|
||||
## Configmap
|
||||
configmap:
|
||||
DOLPHINSCHEDULER_OPTS: ""
|
||||
LOGGER_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
WORKER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
|
||||
WORKER_EXEC_THREADS: "100"
|
||||
WORKER_HEARTBEAT_INTERVAL: "10"
|
||||
WORKER_MAX_CPULOAD_AVG: "100"
|
||||
WORKER_RESERVED_MEMORY: "0.1"
|
||||
WORKER_LISTEN_PORT: "1234"
|
||||
WORKER_GROUPS: "default"
|
||||
WORKER_HOST_WEIGHT: "100"
|
||||
WORKER_MAX_CPULOAD_AVG: "-1"
|
||||
WORKER_RESERVED_MEMORY: "0.3"
|
||||
WORKER_GROUPS: "default"
|
||||
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
|
||||
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
livenessProbe:
|
||||
|
|
@ -275,9 +296,9 @@ alert:
|
|||
# requests:
|
||||
# memory: "1Gi"
|
||||
# cpu: "500m"
|
||||
## ConfigMap
|
||||
## Configmap
|
||||
configmap:
|
||||
DOLPHINSCHEDULER_OPTS: ""
|
||||
ALERT_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
ALERT_PLUGIN_DIR: "lib/plugin/alert"
|
||||
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
|
||||
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
|
|
@ -338,9 +359,9 @@ api:
|
|||
# requests:
|
||||
# memory: "1Gi"
|
||||
# cpu: "500m"
|
||||
## ConfigMap
|
||||
## Configmap
|
||||
configmap:
|
||||
DOLPHINSCHEDULER_OPTS: ""
|
||||
API_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
|
||||
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
|
||||
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
livenessProbe:
|
||||
|
|
@ -367,6 +388,22 @@ api:
|
|||
- "ReadWriteOnce"
|
||||
storageClassName: "-"
|
||||
storage: "20Gi"
|
||||
service:
|
||||
## type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer
|
||||
type: "ClusterIP"
|
||||
## clusterIP is the IP address of the service and is usually assigned randomly by the master
|
||||
clusterIP: ""
|
||||
## nodePort is the port on each node on which this service is exposed when type=NodePort
|
||||
nodePort: ""
|
||||
## externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service
|
||||
externalIPs: []
|
||||
## externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service, requires Type to be ExternalName
|
||||
externalName: ""
|
||||
## loadBalancerIP when service.type is LoadBalancer. LoadBalancer will get created with the IP specified in this field
|
||||
loadBalancerIP: ""
|
||||
## annotations may need to be set when service.type is LoadBalancer
|
||||
## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT
|
||||
annotations: {}
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public class Constants {
|
|||
public static final String ALERT_PROPERTIES_PATH = "/alert.properties";
|
||||
|
||||
/** default alert plugin dir **/
|
||||
public static final String ALERT_PLUGIN_PATH = "./lib/plugin/alert";
|
||||
public static final String ALERT_PLUGIN_PATH = "lib/plugin/alert";
|
||||
|
||||
public static final int ALERT_SCAN_INTERVAL = 5000;
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
#eg : Alert Server Listener port
|
||||
|
||||
#alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server .
|
||||
#alert.plugin.dir=./lib/plugin/alert
|
||||
#alert.plugin.dir=lib/plugin/alert
|
||||
|
||||
#maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ server.jetty.max-http-form-post-size=5000000
|
|||
# messages encoding
|
||||
spring.messages.encoding=UTF-8
|
||||
|
||||
# i18n classpath folder , file prefix messages， if have many files, use "," seperator
|
||||
# i18n classpath folder, file prefix messages. if have many files, use "," seperator
|
||||
spring.messages.basename=i18n/messages
|
||||
|
||||
# Authentication types (supported types: PASSWORD)
|
||||
|
|
|
|||
|
|
@ -93,11 +93,19 @@ public final class Constants {
|
|||
public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
|
||||
|
||||
|
||||
/**
|
||||
* hadoop configuration
|
||||
*/
|
||||
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
|
||||
|
||||
public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
|
||||
|
||||
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
|
||||
|
||||
/**
|
||||
* yarn.resourcemanager.ha.rm.ids
|
||||
*/
|
||||
public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids";
|
||||
public static final String YARN_RESOURCEMANAGER_HA_XX = "xx";
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -153,7 +161,6 @@ public final class Constants {
|
|||
* development.state
|
||||
*/
|
||||
public static final String DEVELOPMENT_STATE = "development.state";
|
||||
public static final String DEVELOPMENT_STATE_DEFAULT_VALUE = "true";
|
||||
|
||||
/**
|
||||
* sudo enable
|
||||
|
|
@ -266,15 +273,6 @@ public final class Constants {
|
|||
public static final String AT_SIGN = "@";
|
||||
|
||||
|
||||
public static final String WORKER_MAX_CPULOAD_AVG = "worker.max.cpuload.avg";
|
||||
|
||||
public static final String WORKER_RESERVED_MEMORY = "worker.reserved.memory";
|
||||
|
||||
public static final String MASTER_MAX_CPULOAD_AVG = "master.max.cpuload.avg";
|
||||
|
||||
public static final String MASTER_RESERVED_MEMORY = "master.reserved.memory";
|
||||
|
||||
|
||||
/**
|
||||
* date format of yyyy-MM-dd HH:mm:ss
|
||||
*/
|
||||
|
|
@ -473,15 +471,6 @@ public final class Constants {
|
|||
*/
|
||||
public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate";
|
||||
|
||||
/**
|
||||
* hadoop configuration
|
||||
*/
|
||||
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
|
||||
|
||||
public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
|
||||
|
||||
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
|
||||
|
||||
|
||||
/**
|
||||
* data source config
|
||||
|
|
@ -834,11 +823,6 @@ public final class Constants {
|
|||
*/
|
||||
public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username";
|
||||
|
||||
/**
|
||||
* default worker group id
|
||||
*/
|
||||
public static final int DEFAULT_WORKER_ID = -1;
|
||||
|
||||
/**
|
||||
* loginUserFromKeytab path
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory;
|
|||
*/
|
||||
public class NetUtils {
|
||||
|
||||
private static final Pattern STS_PATTERN = Pattern.compile("-\\d+$"); // StatefulSet pattern
|
||||
private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$");
|
||||
private static final String NETWORK_PRIORITY_DEFAULT = "default";
|
||||
private static final String NETWORK_PRIORITY_INNER = "inner";
|
||||
|
|
@ -81,13 +80,8 @@ public class NetUtils {
|
|||
if (inetAddress != null) {
|
||||
if (Constants.KUBERNETES_MODE) {
|
||||
String canonicalHost = inetAddress.getCanonicalHostName();
|
||||
if (!canonicalHost.contains(".") || IP_PATTERN.matcher(canonicalHost).matches()) {
|
||||
String host = inetAddress.getHostName();
|
||||
if (STS_PATTERN.matcher(host).find()) {
|
||||
return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless"));
|
||||
}
|
||||
} else if (canonicalHost.contains(".")) {
|
||||
String[] items = canonicalHost.split("\\.");
|
||||
String[] items = canonicalHost.split("\\.");
|
||||
if (items.length == 6 && "svc".equals(items[3])) {
|
||||
return String.format("%s.%s", items[0], items[1]);
|
||||
}
|
||||
return canonicalHost;
|
||||
|
|
|
|||
|
|
@ -15,16 +15,16 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# resource storage type : HDFS, S3, NONE
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
data.basedir.path=/tmp/dolphinscheduler
|
||||
|
||||
# resource storage type: HDFS, S3, NONE
|
||||
resource.storage.type=NONE
|
||||
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resource.upload.path=/dolphinscheduler
|
||||
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
#data.basedir.path=/tmp/dolphinscheduler
|
||||
|
||||
# whether kerberos starts
|
||||
# whether to startup kerberos
|
||||
hadoop.security.authentication.startup.state=false
|
||||
|
||||
# java.security.krb5.conf path
|
||||
|
|
@ -36,13 +36,16 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
|||
# login user from keytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
|
||||
#resource.view.suffixs
|
||||
# kerberos expire time, the unit is hour
|
||||
kerberos.expire.time=2
|
||||
|
||||
# resource view suffixs
|
||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
||||
|
||||
# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path
|
||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
||||
hdfs.root.user=hdfs
|
||||
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
fs.defaultFS=hdfs://mycluster:8020
|
||||
|
||||
# if resource.storage.type=S3, s3 endpoint
|
||||
|
|
@ -54,31 +57,32 @@ fs.s3a.access.key=A3DXS30FO22544RE
|
|||
# if resource.storage.type=S3, s3 secret key
|
||||
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
|
||||
|
||||
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
|
||||
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.
|
||||
yarn.application.status.address=http://10.172.33.1:%s/ws/v1/cluster/apps/%s
|
||||
|
||||
# if custom you resourcemanager port ,you need to replace 8088 else default value.
|
||||
# resourcemanager port, the default value is 8088 if not specified
|
||||
resource.manager.httpaddress.port=8088
|
||||
|
||||
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000)
|
||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
|
||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
||||
yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
|
||||
|
||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
||||
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
|
||||
|
||||
# system env path, If you want to set your own path, you need to set this env file to an absolute path
|
||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
||||
development.state=false
|
||||
|
||||
# kerberos tgt expire time, unit is hours
|
||||
kerberos.expire.time=2
|
||||
# datasource encryption enable
|
||||
datasource.encryption.enable=false
|
||||
|
||||
# datasource encryption salt
|
||||
datasource.encryption.enable=false
|
||||
datasource.encryption.salt=!@#$%^&*
|
||||
|
||||
# Network IP gets priority, default inner outer
|
||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
|
||||
sudo.enable=true
|
||||
|
||||
# network IP gets priority, default: inner outer
|
||||
#dolphin.scheduler.network.priority.strategy=default
|
||||
|
||||
# use sudo or not, if set true ,executing user is tenant user and deploy user need sudo permissions ; if set false, executing user is the deploy user, don't need sudo permissions.
|
||||
sudo.enable=true
|
||||
# system env path
|
||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
||||
|
||||
# development state
|
||||
development.state=false
|
||||
|
|
|
|||
|
|
@ -51,11 +51,24 @@ public class NetUtilsTest {
|
|||
assertEquals("172.17.0.15", NetUtils.getHost(address));
|
||||
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
|
||||
assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address));
|
||||
|
||||
address = mock(InetAddress.class);
|
||||
when(address.getCanonicalHostName()).thenReturn("busybox-1.default-subdomain.my-namespace.svc.cluster-domain.example");
|
||||
when(address.getHostName()).thenReturn("busybox-1");
|
||||
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
|
||||
assertEquals("busybox-1.default-subdomain", NetUtils.getHost(address));
|
||||
|
||||
address = mock(InetAddress.class);
|
||||
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler.cluster-domain.example");
|
||||
when(address.getHostName()).thenReturn("dolphinscheduler");
|
||||
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
|
||||
assertEquals("dolphinscheduler.cluster-domain.example", NetUtils.getHost(address));
|
||||
|
||||
address = mock(InetAddress.class);
|
||||
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0");
|
||||
when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
|
||||
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
|
||||
assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address));
|
||||
assertEquals("dolphinscheduler-worker-0", NetUtils.getHost(address));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@
|
|||
# postgresql
|
||||
spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
|
||||
spring.datasource.username=test
|
||||
spring.datasource.password=test
|
||||
spring.datasource.username=root
|
||||
spring.datasource.password=root
|
||||
|
||||
# mysql
|
||||
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
|
||||
#spring.datasource.username=xxxx
|
||||
#spring.datasource.password=xxxx
|
||||
#spring.datasource.username=ds_user
|
||||
#spring.datasource.password=dolphinscheduler
|
||||
|
||||
# connection configuration
|
||||
#spring.datasource.initialSize=5
|
||||
|
|
@ -66,4 +66,4 @@ spring.datasource.password=test
|
|||
|
||||
# open PSCache, specify count PSCache for every connection
|
||||
#spring.datasource.poolPreparedStatements=true
|
||||
#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
|
||||
#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.server.master.config;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.PropertySource;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
|
@ -25,21 +27,27 @@ import org.springframework.stereotype.Component;
|
|||
@PropertySource(value = "master.properties")
|
||||
public class MasterConfig {
|
||||
|
||||
@Value("${master.listen.port:5678}")
|
||||
private int listenPort;
|
||||
|
||||
@Value("${master.exec.threads:100}")
|
||||
private int masterExecThreads;
|
||||
|
||||
@Value("${master.exec.task.num:20}")
|
||||
private int masterExecTaskNum;
|
||||
|
||||
@Value("${master.dispatch.task.num:3}")
|
||||
private int masterDispatchTaskNumber;
|
||||
|
||||
@Value("${master.host.selector:LowerWeight}")
|
||||
private String hostSelector;
|
||||
|
||||
@Value("${master.heartbeat.interval:10}")
|
||||
private int masterHeartbeatInterval;
|
||||
|
||||
@Value("${master.task.commit.retryTimes:5}")
|
||||
private int masterTaskCommitRetryTimes;
|
||||
|
||||
@Value("${master.dispatch.task.num:3}")
|
||||
private int masterDispatchTaskNumber;
|
||||
|
||||
@Value("${master.task.commit.interval:1000}")
|
||||
private int masterTaskCommitInterval;
|
||||
|
||||
|
|
@ -49,12 +57,6 @@ public class MasterConfig {
|
|||
@Value("${master.reserved.memory:0.3}")
|
||||
private double masterReservedMemory;
|
||||
|
||||
@Value("${master.host.selector:lowerWeight}")
|
||||
private String hostSelector;
|
||||
|
||||
@Value("${master.listen.port:5678}")
|
||||
private int listenPort;
|
||||
|
||||
public int getListenPort() {
|
||||
return listenPort;
|
||||
}
|
||||
|
|
@ -112,7 +114,7 @@ public class MasterConfig {
|
|||
}
|
||||
|
||||
public double getMasterMaxCpuloadAvg() {
|
||||
if (masterMaxCpuloadAvg == -1){
|
||||
if (masterMaxCpuloadAvg == -1) {
|
||||
return Constants.DEFAULT_MASTER_CPU_LOAD;
|
||||
}
|
||||
return masterMaxCpuloadAvg;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ import org.springframework.stereotype.Component;
|
|||
@PropertySource(value = "worker.properties")
|
||||
public class WorkerConfig {
|
||||
|
||||
@Value("${worker.listen.port:1234}")
|
||||
private int listenPort;
|
||||
|
||||
@Value("${worker.exec.threads:100}")
|
||||
private int workerExecThreads;
|
||||
|
||||
|
|
@ -47,9 +50,6 @@ public class WorkerConfig {
|
|||
@Value("${worker.reserved.memory:0.3}")
|
||||
private double workerReservedMemory;
|
||||
|
||||
@Value("${worker.listen.port:1234}")
|
||||
private int listenPort;
|
||||
|
||||
@Value("#{'${worker.groups:default}'.split(',')}")
|
||||
private Set<String> workerGroups;
|
||||
|
||||
|
|
|
|||
|
|
@ -14,23 +14,29 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.server.worker.task;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.StringUtils;
|
||||
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* python command executor
|
||||
|
|
@ -46,7 +52,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
|
|||
* python
|
||||
*/
|
||||
public static final String PYTHON = "python";
|
||||
|
||||
private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$");
|
||||
|
||||
/**
|
||||
* constructor
|
||||
|
|
@ -115,54 +121,64 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
|
|||
@Override
|
||||
protected String commandInterpreter() {
|
||||
String pythonHome = getPythonHome(taskExecutionContext.getEnvFile());
|
||||
if (StringUtils.isEmpty(pythonHome)){
|
||||
return getPythonCommand(pythonHome);
|
||||
}
|
||||
|
||||
/**
|
||||
* get python command
|
||||
*
|
||||
* @param pythonHome python home
|
||||
* @return python command
|
||||
*/
|
||||
public static String getPythonCommand(String pythonHome) {
|
||||
if (StringUtils.isEmpty(pythonHome)) {
|
||||
return PYTHON;
|
||||
}
|
||||
File file = new File(pythonHome);
|
||||
if (file.exists() && file.isFile()) {
|
||||
return pythonHome;
|
||||
}
|
||||
if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) {
|
||||
return pythonHome;
|
||||
}
|
||||
return pythonHome + "/bin/python";
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* get the absolute path of the Python are installed
|
||||
* note :
|
||||
*
|
||||
* for example :
|
||||
* your PYTHON_HOM is /opt/python3.7
|
||||
* get python home
|
||||
*
|
||||
* @param envPath env path
|
||||
* @return python home
|
||||
*/
|
||||
private static String getPythonHome(String envPath){
|
||||
public static String getPythonHome(String envPath) {
|
||||
BufferedReader br = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath)));
|
||||
String line;
|
||||
while ((line = br.readLine()) != null){
|
||||
if (line.contains(Constants.PYTHON_HOME)){
|
||||
while ((line = br.readLine()) != null) {
|
||||
if (line.contains(Constants.PYTHON_HOME)) {
|
||||
sb.append(line);
|
||||
break;
|
||||
}
|
||||
}
|
||||
String result = sb.toString();
|
||||
if (org.apache.commons.lang.StringUtils.isEmpty(result)){
|
||||
if (StringUtils.isEmpty(result)) {
|
||||
return null;
|
||||
}
|
||||
String[] arrs = result.split(Constants.EQUAL_SIGN);
|
||||
if (arrs.length == 2){
|
||||
if (arrs.length == 2) {
|
||||
return arrs[1];
|
||||
}
|
||||
|
||||
}catch (IOException e){
|
||||
logger.error("read file failure",e);
|
||||
}finally {
|
||||
} catch (IOException e) {
|
||||
logger.error("read file failure", e);
|
||||
} finally {
|
||||
try {
|
||||
if (br != null){
|
||||
if (br != null) {
|
||||
br.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(),e);
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ deployUser="dolphinscheduler"
|
|||
# Note: find and load the Alert Plugin Jar from this dir.
|
||||
alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
|
||||
|
||||
# resource storage type：HDFS,S3,NONE
|
||||
# resource storage type: HDFS, S3, NONE
|
||||
resourceStorageType="NONE"
|
||||
|
||||
# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
|
||||
|
|
@ -66,13 +66,16 @@ s3Endpoint="http://192.168.xx.xx:9010"
|
|||
s3AccessKey="xxxxxxxxxx"
|
||||
s3SecretKey="xxxxxxxxxx"
|
||||
|
||||
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
|
||||
# resourcemanager port, the default value is 8088 if not specified
|
||||
resourceManagerHttpAddressPort=8088
|
||||
|
||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
||||
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
|
||||
|
||||
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.
|
||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
||||
singleYarnIp="yarnIp1"
|
||||
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended
|
||||
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resourceUploadPath="/dolphinscheduler"
|
||||
|
||||
# who have permissions to create directory under HDFS/S3 root path
|
||||
|
|
|
|||
|
|
@ -15,29 +15,32 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# master execute thread number
|
||||
# master listen port
|
||||
#master.listen.port=5678
|
||||
|
||||
# master execute thread number to limit process instances in parallel
|
||||
#master.exec.threads=100
|
||||
|
||||
# master execute task number in parallel
|
||||
# master execute task number in parallel per process instance
|
||||
#master.exec.task.num=20
|
||||
|
||||
# master dispatch task number
|
||||
# master dispatch task number per batch
|
||||
#master.dispatch.task.num=3
|
||||
|
||||
# master heartbeat interval
|
||||
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight
|
||||
#master.host.selector=LowerWeight
|
||||
|
||||
# master heartbeat interval, the unit is second
|
||||
#master.heartbeat.interval=10
|
||||
|
||||
# master commit task retry times
|
||||
#master.task.commit.retryTimes=5
|
||||
|
||||
# master commit task interval
|
||||
# master commit task interval, the unit is millisecond
|
||||
#master.task.commit.interval=1000
|
||||
|
||||
# only less than cpu avg load, master server can work. default value -1 : the number of cpu cores * 2
|
||||
# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
|
||||
#master.max.cpuload.avg=-1
|
||||
|
||||
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
|
||||
# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
|
||||
#master.reserved.memory=0.3
|
||||
|
||||
# master listen port
|
||||
#master.listen.port=5678
|
||||
|
|
|
|||
|
|
@ -15,10 +15,13 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# worker execute thread number
|
||||
# worker listener port
|
||||
#worker.listen.port=1234
|
||||
|
||||
# worker execute thread number to limit task instances in parallel
|
||||
#worker.exec.threads=100
|
||||
|
||||
# worker heartbeat interval
|
||||
# worker heartbeat interval, the unit is second
|
||||
#worker.heartbeat.interval=10
|
||||
|
||||
# worker host weight to dispatch tasks, default value 100
|
||||
|
|
@ -27,17 +30,14 @@
|
|||
# worker tenant auto create
|
||||
#worker.tenant.auto.create=false
|
||||
|
||||
# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2
|
||||
# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
|
||||
#worker.max.cpuload.avg=-1
|
||||
|
||||
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
|
||||
# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
|
||||
#worker.reserved.memory=0.3
|
||||
|
||||
# worker listener port
|
||||
#worker.listen.port=1234
|
||||
|
||||
# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test'
|
||||
# default worker groups separated by comma, like 'worker.groups=default,test'
|
||||
#worker.groups=default
|
||||
|
||||
# alert server listener host
|
||||
alert.listen.host=localhost
|
||||
# alert server listen host
|
||||
#alert.listen.host=localhost
|
||||
|
|
|
|||
|
|
@ -1,82 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.server.worker;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.StringUtils;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class EnvFileTest {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(EnvFileTest.class);
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
String path = System.getProperty("user.dir")+"/script/env/dolphinscheduler_env.sh";
|
||||
String pythonHome = getPythonHome(path);
|
||||
logger.info(pythonHome);
|
||||
}
|
||||
|
||||
/**
|
||||
* get python home
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
private static String getPythonHome(String path){
|
||||
BufferedReader br = null;
|
||||
String line = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
|
||||
while ((line = br.readLine()) != null){
|
||||
if (line.contains(Constants.PYTHON_HOME)) {
|
||||
sb.append(line);
|
||||
break;
|
||||
}
|
||||
}
|
||||
String result = sb.toString();
|
||||
if (StringUtils.isEmpty(result)){
|
||||
return null;
|
||||
}
|
||||
String[] arrs = result.split("=");
|
||||
if (arrs.length == 2){
|
||||
return arrs[1];
|
||||
}
|
||||
|
||||
}catch (IOException e){
|
||||
logger.error("read file failed",e);
|
||||
}finally {
|
||||
try {
|
||||
if (br != null){
|
||||
br.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(),e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.server.worker.task;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class PythonCommandExecutorTest {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PythonCommandExecutorTest.class);
|
||||
|
||||
@Test
|
||||
public void testGetPythonHome() {
|
||||
String path = System.getProperty("user.dir") + "/script/env/dolphinscheduler_env.sh";
|
||||
if (path.contains("dolphinscheduler-server/")) {
|
||||
path = path.replace("dolphinscheduler-server/", "");
|
||||
}
|
||||
String pythonHome = PythonCommandExecutor.getPythonHome(path);
|
||||
logger.info(pythonHome);
|
||||
Assert.assertNotNull(pythonHome);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetPythonCommand() {
|
||||
String pythonCommand = PythonCommandExecutor.getPythonCommand(null);
|
||||
Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("");
|
||||
Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/bin/python");
|
||||
Assert.assertEquals("/usr/bin/python", pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/local/bin/python2");
|
||||
Assert.assertEquals("/usr/local/bin/python2", pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/python/bin/python3.8");
|
||||
Assert.assertEquals("/opt/python/bin/python3.8", pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python");
|
||||
Assert.assertEquals("/opt/soft/python/bin/python", pythonCommand);
|
||||
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python-3.8");
|
||||
Assert.assertEquals("/opt/soft/python-3.8/bin/python", pythonCommand);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -43,8 +43,10 @@ sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.propert
|
|||
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties
|
||||
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties
|
||||
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties
|
||||
sed -i ${txt} "s#resource.manager.httpaddress.port.*#resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}#g" conf/common.properties
|
||||
sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties
|
||||
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties
|
||||
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s#g" conf/common.properties
|
||||
sed -i ${txt} "s#yarn.job.history.status.address.*#yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s#g" conf/common.properties
|
||||
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties
|
||||
sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties
|
||||
sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties
|
||||
|
|
@ -56,9 +58,9 @@ sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeep
|
|||
sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties
|
||||
sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
|
||||
|
||||
sed -i ${txt} "s#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties
|
||||
sed -i ${txt} "s#\#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties
|
||||
|
||||
sed -i ${txt} "s#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties
|
||||
sed -i ${txt} "s#\#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties
|
||||
|
||||
# 2.create directory
|
||||
echo "2.create directory"
|
||||
|
|
|
|||
3
pom.xml
3
pom.xml
|
|
@ -970,7 +970,6 @@
|
|||
<include>**/server/worker/shell/ShellCommandExecutorTest.java</include>
|
||||
<include>**/server/worker/sql/SqlExecutorTest.java</include>
|
||||
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
|
||||
<include>**/server/worker/task/EnvFileTest.java</include>
|
||||
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
|
||||
<include>**/server/worker/task/datax/DataxTaskTest.java</include>
|
||||
<!--<include>**/server/worker/task/http/HttpTaskTest.java</include>-->
|
||||
|
|
@ -979,9 +978,9 @@
|
|||
<include>**/server/worker/task/shell/ShellTaskTest.java</include>
|
||||
<include>**/server/worker/task/TaskManagerTest.java</include>
|
||||
<include>**/server/worker/task/AbstractCommandExecutorTest.java</include>
|
||||
<include>**/server/worker/task/PythonCommandExecutorTest.java</include>
|
||||
<include>**/server/worker/task/ShellTaskReturnTest.java</include>
|
||||
<include>**/server/worker/task/sql/SqlTaskTest.java</include>
|
||||
<include>**/server/worker/EnvFileTest.java</include>
|
||||
<include>**/server/worker/runner/TaskExecuteThreadTest.java</include>
|
||||
<include>**/server/worker/runner/WorkerManagerThreadTest.java</include>
|
||||
<include>**/service/quartz/cron/CronUtilsTest.java</include>
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
|
|||
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
|
||||
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
|
||||
|
||||
export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
|
||||
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
|
||||
export STOP_TIMEOUT=5
|
||||
|
||||
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
|
||||
|
|
|
|||
|
|
@ -29,10 +29,11 @@ shift
|
|||
command=$1
|
||||
shift
|
||||
|
||||
echo "Begin $startStop $command......"
|
||||
|
||||
BIN_DIR=`dirname $0`
|
||||
BIN_DIR=`cd "$BIN_DIR"; pwd`
|
||||
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
|
||||
DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
|
||||
|
||||
source /etc/profile
|
||||
|
||||
|
|
@ -56,51 +57,47 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
|
|||
|
||||
cd $DOLPHINSCHEDULER_HOME
|
||||
|
||||
export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
|
||||
|
||||
if [ "$command" = "api-server" ]; then
|
||||
HEAP_INITIAL_SIZE=1g
|
||||
HEAP_MAX_SIZE=1g
|
||||
HEAP_NEW_GENERATION_SIZE=512m
|
||||
LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
|
||||
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
|
||||
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
|
||||
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
|
||||
elif [ "$command" = "master-server" ]; then
|
||||
HEAP_INITIAL_SIZE=4g
|
||||
HEAP_MAX_SIZE=4g
|
||||
HEAP_NEW_GENERATION_SIZE=2g
|
||||
LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false"
|
||||
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
|
||||
HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
|
||||
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
|
||||
elif [ "$command" = "worker-server" ]; then
|
||||
HEAP_INITIAL_SIZE=2g
|
||||
HEAP_MAX_SIZE=2g
|
||||
HEAP_NEW_GENERATION_SIZE=1g
|
||||
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false"
|
||||
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
|
||||
HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
|
||||
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
|
||||
elif [ "$command" = "alert-server" ]; then
|
||||
HEAP_INITIAL_SIZE=1g
|
||||
HEAP_MAX_SIZE=1g
|
||||
HEAP_NEW_GENERATION_SIZE=512m
|
||||
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
|
||||
CLASS=org.apache.dolphinscheduler.alert.AlertServer
|
||||
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
|
||||
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
|
||||
elif [ "$command" = "logger-server" ]; then
|
||||
HEAP_INITIAL_SIZE=1g
|
||||
HEAP_MAX_SIZE=1g
|
||||
HEAP_NEW_GENERATION_SIZE=512m
|
||||
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
|
||||
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
|
||||
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
|
||||
elif [ "$command" = "zookeeper-server" ]; then
|
||||
#note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server
|
||||
LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
|
||||
CLASS=org.apache.dolphinscheduler.service.zk.ZKServer
|
||||
else
|
||||
echo "Error: No command named \`$command' was found."
|
||||
echo "Error: No command named '$command' was found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
|
||||
|
||||
case $startStop in
|
||||
(start)
|
||||
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
|
||||
if [ "$DOCKER" = "true" ]; then
|
||||
echo "start in docker"
|
||||
echo start $command in docker
|
||||
export DOLPHINSCHEDULER_OPTS="$DOLPHINSCHEDULER_OPTS -XX:-UseContainerSupport"
|
||||
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
|
||||
$JAVA_HOME/bin/java $exec_command
|
||||
else
|
||||
[ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
|
||||
|
|
@ -113,6 +110,7 @@ case $startStop in
|
|||
fi
|
||||
|
||||
echo starting $command, logging to $log
|
||||
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
|
||||
echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
|
||||
nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
|
||||
echo $! > $pid
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
|
|||
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
|
||||
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
|
||||
|
||||
export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
|
||||
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
|
||||
export STOP_TIMEOUT=5
|
||||
|
||||
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler
|
||||
|
|
|
|||
Loading…
Reference in New Issue