Merge remote-tracking branch 'ikiler/fix-dinky-params' into fix-dinky-params
This commit is contained in:
ikiler
commit
b799789edf
|
|
@ -49,4 +49,4 @@ github:
|
|||
- "Mergeable: milestone-label-check"
|
||||
required_pull_request_reviews:
|
||||
dismiss_stale_reviews: true
|
||||
required_approving_review_count: 1
|
||||
required_approving_review_count: 2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
name: DSIP
|
||||
description: Suggest an idea for this project
|
||||
title: "[DSIP-][Module Name] DSIP title"
|
||||
labels: [ "DSIP", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first
|
||||
to see whether the same DSIP was created already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no
|
||||
similar DSIP.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Motivation
|
||||
description: Why you want to do this change?
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Design Detail
|
||||
description: Your design.
|
||||
placeholder: >
|
||||
It's better to provide a detailed design, such as the design of the interface, the design of the database, etc.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Compatibility, Deprecation, and Migration Plan
|
||||
description: >
|
||||
If this feature is related to compatibility, deprecation, or migration, please describe it here.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Test Plan
|
||||
description: >
|
||||
How to test this improvement.
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -106,10 +106,14 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: cluster-test-mysql
|
||||
script: .github/workflows/cluster-test/mysql/start-job.sh
|
||||
- name: cluster-test-postgresql
|
||||
script: .github/workflows/cluster-test/postgresql/start-job.sh
|
||||
- name: cluster-test-mysql-with-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-mysql-with-mysql-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-with-postgresql-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
|
|||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
|
|
@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME
|
|||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Create database
|
||||
mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
34
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
vendored
Normal file
34
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mysql:
|
||||
container_name: mysql
|
||||
image: mysql:5.7.36
|
||||
command: --default-authentication-plugin=mysql_native_password
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: 123456
|
||||
ports:
|
||||
- "3306:3306"
|
||||
healthcheck:
|
||||
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
57
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
vendored
Executable file
57
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
vendored
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-mysql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export SPRING_DATASOURCE_USERNAME=root
|
||||
export SPRING_DATASOURCE_PASSWORD=123456
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=root
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=123456
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
|
@ -18,16 +18,16 @@
|
|||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile .
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
|
||||
RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
29
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
29
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_mysql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
108
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
vendored
Normal file
108
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
vendored
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
39
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
vendored
Normal file
39
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
41
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
vendored
Normal file
41
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
35
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
vendored
Normal file
35
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
container_name: postgres
|
||||
image: postgres:14.1
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dolphinscheduler
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
||||
57
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
vendored
Normal file
57
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
vendored
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export SPRING_DATASOURCE_USERNAME=postgres
|
||||
export SPRING_DATASOURCE_PASSWORD=postgres
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=jdbc
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=postgres
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
58
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
vendored
Normal file
58
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
33
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
vendored
Normal file
33
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
|
|||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
29
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
29
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_postgresql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
109
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
vendored
Normal file
109
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
vendored
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -18,16 +18,16 @@
|
|||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile .
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -174,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| image.master | string | `"dolphinscheduler-master"` | master image |
|
||||
| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent |
|
||||
| image.pullSecret | string | `""` | Specify a imagePullSecrets |
|
||||
| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler |
|
||||
| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler |
|
||||
| image.tag | string | `"latest"` | Docker image version for the DolphinScheduler |
|
||||
| image.tools | string | `"dolphinscheduler-tools"` | tools image |
|
||||
| image.worker | string | `"dolphinscheduler-worker"` | worker image |
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ initImage:
|
|||
|
||||
image:
|
||||
# -- Docker image repository for the DolphinScheduler
|
||||
registry: apache/dolphinscheduler
|
||||
registry: apache
|
||||
# -- Docker image version for the DolphinScheduler
|
||||
tag: latest
|
||||
# -- Image pull policy. Options: Always, Never, IfNotPresent
|
||||
|
|
|
|||
|
|
@ -286,7 +286,6 @@ Location: `master-server/conf/application.yaml`
|
|||
| Parameters | Default value | Description |
|
||||
|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master listen port |
|
||||
| master.fetch-command-num | 10 | the number of commands fetched by master |
|
||||
| master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel |
|
||||
| master.exec-threads | 100 | master execute thread number to limit process instances in parallel |
|
||||
| master.dispatch-task-number | 3 | master dispatch task number per batch |
|
||||
|
|
@ -305,6 +304,9 @@ Location: `master-server/conf/application.yaml`
|
|||
| master.registry-disconnect-strategy.strategy | stop | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely |
|
||||
| master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory |
|
||||
| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` |
|
||||
| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db |
|
||||
| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master |
|
||||
|
||||
### Worker Server related configuration
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Use the command line mode `cd` enter the `dolphinscheduler-ui` project director
|
|||
> If `npm install` is very slow, you can set the taobao mirror
|
||||
|
||||
```
|
||||
npm config set registry http://registry.npm.taobao.org/
|
||||
npm config set registry http://registry.npmmirror.com/
|
||||
```
|
||||
|
||||
- Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend:
|
||||
|
|
|
|||
|
|
@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory
|
|||
sudo rm -rf node_modules
|
||||
```
|
||||
|
||||
2, install node-sass through npm.taobao.org
|
||||
2, install node-sass through npmmirror.com
|
||||
|
||||
```
|
||||
sudo npm uninstall node-sass
|
||||
sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
|
||||
sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
|
||||
```
|
||||
|
||||
3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md)
|
||||
|
|
|
|||
|
|
@ -14,16 +14,15 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
|
|||
|
||||
## Install DolphinScheduler
|
||||
|
||||
Please download the source code package `apache-dolphinscheduler-<version>-src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download)
|
||||
|
||||
To publish the release name `dolphinscheduler` version, please execute the following commands:
|
||||
|
||||
```
|
||||
$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
|
||||
$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler . --set image.tag=<version>
|
||||
```bash
|
||||
# Choose the corresponding version yourself
|
||||
export VERSION=3.2.1
|
||||
helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
|
||||
tar -xvf dolphinscheduler-helm-${VERSION}.tgz
|
||||
cd dolphinscheduler-helm
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm dependency update .
|
||||
helm install dolphinscheduler .
|
||||
```
|
||||
|
||||
To publish the release name `dolphinscheduler` version to `test` namespace:
|
||||
|
|
|
|||
|
|
@ -123,7 +123,6 @@ export SPRING_DATASOURCE_PASSWORD={password}
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ Spark task type for executing Spark application. When executing the Spark task,
|
|||
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Program type | Supports Java, Scala, Python, and SQL. |
|
||||
| The class of main function | The **full path** of Main Class, the entry point of the Spark program. |
|
||||
| Master | The The master URL for the cluster. |
|
||||
| Main jar package | The Spark jar package (upload by Resource Center). |
|
||||
| SQL scripts | SQL statements in .sql files that Spark sql runs. |
|
||||
| Deployment mode | <ul><li>spark submit supports three modes: cluster, client and local.</li><li>spark sql supports client and local modes.</li></ul> |
|
||||
|
|
|
|||
|
|
@ -281,29 +281,30 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
|
||||
位置：`master-server/conf/application.yaml`
|
||||
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------------------------------------|--------------|-----------------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master监听端口 |
|
||||
| master.fetch-command-num | 10 | master拉取command数量 |
|
||||
| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command |
|
||||
| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 |
|
||||
| master.dispatch-task-number | 3 | master每个批次的派发任务数量 |
|
||||
| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight |
|
||||
| master.max-heartbeat-interval | 10s | master最大心跳间隔 |
|
||||
| master.task-commit-retry-times | 5 | 任务重试次数 |
|
||||
| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 |
|
||||
| master.state-wheel-interval | 5 | 轮询检查状态时间 |
|
||||
| master.server-load-protection.enabled | true | 是否开启系统保护策略 |
|
||||
| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU |
|
||||
| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU |
|
||||
| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 |
|
||||
| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
|
||||
| master.failover-interval | 10 | failover间隔，单位为分钟 |
|
||||
| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application |
|
||||
| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, |
|
||||
| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
|
||||
| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 |
|
||||
| 参数 | 默认值 | 描述 |
|
||||
|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| master.listen-port | 5678 | master监听端口 |
|
||||
| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command |
|
||||
| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 |
|
||||
| master.dispatch-task-number | 3 | master每个批次的派发任务数量 |
|
||||
| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight |
|
||||
| master.max-heartbeat-interval | 10s | master最大心跳间隔 |
|
||||
| master.task-commit-retry-times | 5 | 任务重试次数 |
|
||||
| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 |
|
||||
| master.state-wheel-interval | 5 | 轮询检查状态时间 |
|
||||
| master.server-load-protection.enabled | true | 是否开启系统保护策略 |
|
||||
| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU |
|
||||
| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU |
|
||||
| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 |
|
||||
| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
|
||||
| master.failover-interval | 10 | failover间隔，单位为分钟 |
|
||||
| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application |
|
||||
| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting |
|
||||
| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 |
|
||||
| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 |
|
||||
| master.command-fetch-strategy.type | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED` |
|
||||
| master.command-fetch-strategy.config.id-step | 1 | 数据库中t_ds_command的id自增步长 |
|
||||
| master.command-fetch-strategy.config.fetch-size | 10 | master拉取command数量 |
|
||||
|
||||
## Worker Server相关配置
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2
|
|||
> 如果 `npm install` 速度非常慢，你可以设置淘宝镜像
|
||||
|
||||
```
|
||||
npm config set registry http://registry.npm.taobao.org/
|
||||
npm config set registry http://registry.npmmirror.com/
|
||||
```
|
||||
|
||||
- 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互：
|
||||
|
|
|
|||
|
|
@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录
|
|||
sudo rm -rf node_modules
|
||||
```
|
||||
|
||||
2，通过 npm.taobao.org 下载 node-sass
|
||||
2，通过 npmmirror.com 下载 node-sass
|
||||
|
||||
```
|
||||
sudo npm uninstall node-sass
|
||||
sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
|
||||
sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/
|
||||
```
|
||||
|
||||
3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md)
|
||||
|
|
|
|||
|
|
@ -14,16 +14,15 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务
|
|||
|
||||
## 安装 dolphinscheduler
|
||||
|
||||
请下载源码包 apache-dolphinscheduler-<version>-src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download)
|
||||
|
||||
发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令：
|
||||
|
||||
```
|
||||
$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
|
||||
$ cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler . --set image.tag=<version>
|
||||
```bash
|
||||
# 自行选择对应的版本
|
||||
export VERSION=3.2.1
|
||||
helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION}
|
||||
tar -xvf dolphinscheduler-helm-${VERSION}.tgz
|
||||
cd dolphinscheduler-helm
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm dependency update .
|
||||
helm install dolphinscheduler .
|
||||
```
|
||||
|
||||
将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中：
|
||||
|
|
|
|||
|
|
@ -118,7 +118,6 @@ export SPRING_DATASOURCE_PASSWORD={password}
|
|||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ Spark 任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支
|
|||
- 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。
|
||||
- 主函数的 Class：Spark 程序的入口 Main class 的全路径。
|
||||
- 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。
|
||||
- Master：执行 Spark 集群的 Master Url。
|
||||
- SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。
|
||||
- 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。
|
||||
(2) spark sql 支持 client 和 local 两种模式。
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ import java.util.Objects;
|
|||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
|
|
@ -189,7 +191,7 @@ public final class DingTalkSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "UTF-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
@ -317,15 +319,17 @@ public final class DingTalkSender {
|
|||
String sign = org.apache.commons.lang3.StringUtils.EMPTY;
|
||||
try {
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256"));
|
||||
byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8"));
|
||||
sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8");
|
||||
mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
|
||||
byte[] signData = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8));
|
||||
sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), StandardCharsets.UTF_8.name());
|
||||
} catch (Exception e) {
|
||||
log.error("generate sign error, message:{}", e);
|
||||
}
|
||||
return url + "×tamp=" + timestamp + "&sign=" + sign;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
static final class DingTalkSendMsgResponse {
|
||||
|
||||
private Integer errcode;
|
||||
|
|
@ -334,22 +338,6 @@ public final class DingTalkSender {
|
|||
public DingTalkSendMsgResponse() {
|
||||
}
|
||||
|
||||
public Integer getErrcode() {
|
||||
return this.errcode;
|
||||
}
|
||||
|
||||
public void setErrcode(Integer errcode) {
|
||||
this.errcode = errcode;
|
||||
}
|
||||
|
||||
public String getErrmsg() {
|
||||
return this.errmsg;
|
||||
}
|
||||
|
||||
public void setErrmsg(String errmsg) {
|
||||
this.errmsg = errmsg;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o) {
|
||||
if (o == this) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk;
|
|||
|
||||
import org.apache.dolphinscheduler.alert.api.AlertResult;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -47,7 +48,7 @@ public class DingTalkSenderTest {
|
|||
@Test
|
||||
public void testSend() {
|
||||
DingTalkSender dingTalkSender = new DingTalkSender(dingTalkConfig);
|
||||
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8");
|
||||
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", StandardCharsets.UTF_8.name());
|
||||
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true");
|
||||
dingTalkSender = new DingTalkSender(dingTalkConfig);
|
||||
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test");
|
||||
|
|
|
|||
|
|
@ -56,8 +56,6 @@ public final class EmailConstants {
|
|||
|
||||
public static final String TABLE_BODY_HTML_TAIL = "</table></body></html>";
|
||||
|
||||
public static final String UTF_8 = "UTF-8";
|
||||
|
||||
public static final String EXCEL_SUFFIX_XLSX = ".xlsx";
|
||||
|
||||
public static final String SINGLE_SLASH = "/";
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.commons.mail.HtmlEmail;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
|
@ -171,7 +172,7 @@ public final class MailSender {
|
|||
Session session = getSession();
|
||||
email.setMailSession(session);
|
||||
email.setFrom(mailSenderEmail);
|
||||
email.setCharset(EmailConstants.UTF_8);
|
||||
email.setCharset(StandardCharsets.UTF_8.name());
|
||||
if (CollectionUtils.isNotEmpty(receivers)) {
|
||||
// receivers mail
|
||||
for (String receiver : receivers) {
|
||||
|
|
@ -344,7 +345,8 @@ public final class MailSender {
|
|||
ExcelUtils.genExcelFile(content, randomFilename, xlsFilePath);
|
||||
|
||||
part2.attachFile(file);
|
||||
part2.setFileName(MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, EmailConstants.UTF_8, "B"));
|
||||
part2.setFileName(
|
||||
MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, StandardCharsets.UTF_8.name(), "B"));
|
||||
// add components to collection
|
||||
partList.addBodyPart(part1);
|
||||
partList.addBodyPart(part2);
|
||||
|
|
|
|||
|
|
@ -66,8 +66,15 @@ public class ExcelUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testGenExcelFileByCheckDir() {
|
||||
ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls");
|
||||
File file = new File("/tmp/xls" + EmailConstants.SINGLE_SLASH + "t" + EmailConstants.EXCEL_SUFFIX_XLSX);
|
||||
String path = "/tmp/xls";
|
||||
ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", path);
|
||||
File file =
|
||||
new File(
|
||||
path
|
||||
+ EmailConstants.SINGLE_SLASH
|
||||
+ "t"
|
||||
+ EmailConstants.EXCEL_SUFFIX_XLSX);
|
||||
file.delete();
|
||||
Assertions.assertFalse(file.exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,9 @@
|
|||
package org.apache.dolphinscheduler.plugin.alert.email;
|
||||
|
||||
import org.apache.dolphinscheduler.alert.api.AlertConstants;
|
||||
import org.apache.dolphinscheduler.alert.api.AlertResult;
|
||||
import org.apache.dolphinscheduler.alert.api.ShowType;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate;
|
||||
import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -42,7 +41,6 @@ public class MailUtilsTest {
|
|||
private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class);
|
||||
static MailSender mailSender;
|
||||
private static Map<String, String> emailConfig = new HashMap<>();
|
||||
private static AlertTemplate alertTemplate;
|
||||
|
||||
@BeforeAll
|
||||
public static void initEmailConfig() {
|
||||
|
|
@ -59,7 +57,6 @@ public class MailUtilsTest {
|
|||
emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, "347801120@qq.com");
|
||||
emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, "347801120@qq.com");
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp());
|
||||
alertTemplate = new DefaultHTMLTemplate();
|
||||
mailSender = new MailSender(emailConfig);
|
||||
}
|
||||
|
||||
|
|
@ -77,9 +74,10 @@ public class MailUtilsTest {
|
|||
+ "\"Host: 192.168.xx.xx\","
|
||||
+ "\"Notify group :4\"]";
|
||||
|
||||
mailSender.sendMails(
|
||||
AlertResult alertResult = mailSender.sendMails(
|
||||
"Mysql Exception",
|
||||
content);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -108,7 +106,8 @@ public class MailUtilsTest {
|
|||
emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "user");
|
||||
emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd");
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails(title, content);
|
||||
AlertResult alertResult = mailSender.sendMails(title, content);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
}
|
||||
|
||||
public String list2String() {
|
||||
|
|
@ -134,7 +133,6 @@ public class MailUtilsTest {
|
|||
logger.info(mapjson);
|
||||
|
||||
return mapjson;
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -143,7 +141,8 @@ public class MailUtilsTest {
|
|||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails(title, content);
|
||||
AlertResult alertResult = mailSender.sendMails(title, content);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -151,7 +150,8 @@ public class MailUtilsTest {
|
|||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails("gaojing", content);
|
||||
AlertResult alertResult = mailSender.sendMails("gaojing", content);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -159,7 +159,7 @@ public class MailUtilsTest {
|
|||
String content = list2String();
|
||||
emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp());
|
||||
mailSender = new MailSender(emailConfig);
|
||||
mailSender.sendMails("gaojing", content);
|
||||
AlertResult alertResult = mailSender.sendMails("gaojing", content);
|
||||
Assertions.assertEquals("false", alertResult.getStatus());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -161,7 +162,7 @@ public final class FeiShuSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "utf-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import java.net.URI;
|
|||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -58,7 +59,6 @@ public final class HttpSender {
|
|||
* request type get
|
||||
*/
|
||||
private static final String REQUEST_TYPE_GET = "GET";
|
||||
private static final String DEFAULT_CHARSET = "utf-8";
|
||||
private final String headerParams;
|
||||
private final String bodyParams;
|
||||
private final String contentField;
|
||||
|
|
@ -124,7 +124,7 @@ public final class HttpSender {
|
|||
|
||||
CloseableHttpResponse response = httpClient.execute(httpRequest);
|
||||
HttpEntity entity = response.getEntity();
|
||||
return EntityUtils.toString(entity, DEFAULT_CHARSET);
|
||||
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private void createHttpRequest(String msg) throws MalformedURLException, URISyntaxException {
|
||||
|
|
@ -157,7 +157,8 @@ public final class HttpSender {
|
|||
type = URL_SPLICE_CHAR;
|
||||
}
|
||||
try {
|
||||
url = String.format("%s%s%s=%s", url, type, contentField, URLEncoder.encode(msg, DEFAULT_CHARSET));
|
||||
url = String.format("%s%s%s=%s", url, type, contentField,
|
||||
URLEncoder.encode(msg, StandardCharsets.UTF_8.name()));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -190,7 +191,7 @@ public final class HttpSender {
|
|||
}
|
||||
// set msg content field
|
||||
objectNode.put(contentField, msg);
|
||||
StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), DEFAULT_CHARSET);
|
||||
StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), StandardCharsets.UTF_8);
|
||||
((HttpPost) httpRequest).setEntity(entity);
|
||||
} catch (Exception e) {
|
||||
log.error("send http alert msg exception : {}", e.getMessage());
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.http.impl.client.HttpClients;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
@ -90,7 +91,7 @@ public class PrometheusAlertSender {
|
|||
}
|
||||
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "utf-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
log.error(
|
||||
"Prometheus alert manager send alert failed, http status code: {}, title: {} ,content: {}, resp: {}",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public class ProcessUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testExecuteScript() {
|
||||
ProcessUtils.executeScript(cmd);
|
||||
int code = ProcessUtils.executeScript(cmd);
|
||||
assert code != -1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
|||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
|
@ -81,11 +82,11 @@ public final class SlackSender {
|
|||
}
|
||||
|
||||
HttpPost httpPost = new HttpPost(webHookUrl);
|
||||
httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), "UTF-8"));
|
||||
httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), StandardCharsets.UTF_8));
|
||||
CloseableHttpResponse response = httpClient.execute(httpPost);
|
||||
|
||||
HttpEntity entity = response.getEntity();
|
||||
return EntityUtils.toString(entity, "UTF-8");
|
||||
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
log.error("Send message to slack error.", e);
|
||||
return "System Exception";
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ public final class TelegramSender {
|
|||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, "UTF-8");
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
|
|||
|
|
@ -23,8 +23,6 @@ public final class WeChatAlertConstants {
|
|||
|
||||
static final String MARKDOWN_ENTER = "\n";
|
||||
|
||||
static final String CHARSET = "UTF-8";
|
||||
|
||||
static final String WE_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}";
|
||||
|
||||
static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" +
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import org.apache.http.impl.client.HttpClients;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -45,6 +46,8 @@ import java.util.Map;
|
|||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
|
|
@ -85,12 +88,12 @@ public final class WeChatSender {
|
|||
CloseableHttpClient httpClient =
|
||||
HttpClients.custom().setRetryHandler(HttpServiceRetryStrategy.retryStrategy).build()) {
|
||||
HttpPost httpPost = new HttpPost(url);
|
||||
httpPost.setEntity(new StringEntity(data, WeChatAlertConstants.CHARSET));
|
||||
httpPost.setEntity(new StringEntity(data, StandardCharsets.UTF_8));
|
||||
CloseableHttpResponse response = httpClient.execute(httpPost);
|
||||
String resp;
|
||||
try {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET);
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
} finally {
|
||||
response.close();
|
||||
|
|
@ -142,7 +145,7 @@ public final class WeChatSender {
|
|||
HttpGet httpGet = new HttpGet(url);
|
||||
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
|
||||
HttpEntity entity = response.getEntity();
|
||||
resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET);
|
||||
resp = EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
EntityUtils.consume(entity);
|
||||
}
|
||||
|
||||
|
|
@ -259,6 +262,8 @@ public final class WeChatSender {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
static final class WeChatSendMsgResponse {
|
||||
|
||||
private Integer errcode;
|
||||
|
|
@ -267,22 +272,6 @@ public final class WeChatSender {
|
|||
public WeChatSendMsgResponse() {
|
||||
}
|
||||
|
||||
public Integer getErrcode() {
|
||||
return this.errcode;
|
||||
}
|
||||
|
||||
public void setErrcode(Integer errcode) {
|
||||
this.errcode = errcode;
|
||||
}
|
||||
|
||||
public String getErrmsg() {
|
||||
return this.errmsg;
|
||||
}
|
||||
|
||||
public void setErrmsg(String errmsg) {
|
||||
this.errmsg = errmsg;
|
||||
}
|
||||
|
||||
public boolean equals(final Object o) {
|
||||
if (o == this) {
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@
|
|||
<directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
|
||||
<includes>
|
||||
<include>**/*.properties</include>
|
||||
<include>**/*.yaml</include>
|
||||
</includes>
|
||||
<outputDirectory>conf</outputDirectory>
|
||||
</fileSet>
|
||||
|
|
|
|||
|
|
@ -23,25 +23,29 @@ import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient;
|
|||
import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer;
|
||||
import org.apache.dolphinscheduler.alert.service.AlertBootstrapService;
|
||||
import org.apache.dolphinscheduler.alert.service.ListenerEventPostService;
|
||||
import org.apache.dolphinscheduler.common.CommonConfiguration;
|
||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||
import org.apache.dolphinscheduler.dao.DaoConfiguration;
|
||||
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
@SpringBootApplication
|
||||
@ComponentScan("org.apache.dolphinscheduler")
|
||||
@Slf4j
|
||||
@Import({CommonConfiguration.class,
|
||||
DaoConfiguration.class,
|
||||
RegistryConfiguration.class})
|
||||
@SpringBootApplication
|
||||
public class AlertServer {
|
||||
|
||||
@Autowired
|
||||
|
|
@ -59,11 +63,11 @@ public class AlertServer {
|
|||
AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount);
|
||||
Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance());
|
||||
Thread.currentThread().setName(Constants.THREAD_NAME_ALERT_SERVER);
|
||||
new SpringApplicationBuilder(AlertServer.class).run(args);
|
||||
SpringApplication.run(AlertServer.class, args);
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void run(ApplicationReadyEvent readyEvent) {
|
||||
@PostConstruct
|
||||
public void run() {
|
||||
log.info("Alert server is staring ...");
|
||||
alertPluginManager.start();
|
||||
alertRegistryClient.start();
|
||||
|
|
|
|||
|
|
@ -30,15 +30,7 @@ spring:
|
|||
password: root
|
||||
hikari:
|
||||
connection-test-query: select 1
|
||||
minimum-idle: 5
|
||||
auto-commit: true
|
||||
validation-timeout: 3000
|
||||
pool-name: DolphinScheduler
|
||||
maximum-pool-size: 50
|
||||
connection-timeout: 30000
|
||||
idle-timeout: 600000
|
||||
leak-detection-threshold: 0
|
||||
initialization-fail-timeout: 1
|
||||
|
||||
# Mybatis-plus configuration, you don't need to change it
|
||||
mybatis-plus:
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@
|
|||
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
|
||||
<includes>
|
||||
<include>**/*.properties</include>
|
||||
<include>**/*.yaml</include>
|
||||
</includes>
|
||||
<outputDirectory>conf</outputDirectory>
|
||||
</fileSet>
|
||||
|
|
|
|||
|
|
@ -18,12 +18,17 @@
|
|||
package org.apache.dolphinscheduler.api;
|
||||
|
||||
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
|
||||
import org.apache.dolphinscheduler.common.CommonConfiguration;
|
||||
import org.apache.dolphinscheduler.common.enums.PluginType;
|
||||
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
|
||||
import org.apache.dolphinscheduler.dao.DaoConfiguration;
|
||||
import org.apache.dolphinscheduler.dao.PluginDao;
|
||||
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
|
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
|
||||
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
|
||||
import org.apache.dolphinscheduler.service.ServiceConfiguration;
|
||||
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
|
||||
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
|
||||
|
||||
|
|
@ -37,18 +42,19 @@ import org.springframework.boot.SpringApplication;
|
|||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.boot.web.servlet.ServletComponentScan;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.event.EventListener;
|
||||
|
||||
@Slf4j
|
||||
@Import({DaoConfiguration.class,
|
||||
CommonConfiguration.class,
|
||||
ServiceConfiguration.class,
|
||||
StorageConfiguration.class,
|
||||
RegistryConfiguration.class})
|
||||
@ServletComponentScan
|
||||
@SpringBootApplication
|
||||
@ComponentScan("org.apache.dolphinscheduler")
|
||||
@Slf4j
|
||||
public class ApiApplicationServer {
|
||||
|
||||
@Autowired
|
||||
private TaskPluginManager taskPluginManager;
|
||||
|
||||
@Autowired
|
||||
private PluginDao pluginDao;
|
||||
|
||||
|
|
@ -62,8 +68,8 @@ public class ApiApplicationServer {
|
|||
public void run(ApplicationReadyEvent readyEvent) {
|
||||
log.info("Received spring application context ready event will load taskPlugin and write to DB");
|
||||
// install task plugin
|
||||
taskPluginManager.loadPlugin();
|
||||
for (Map.Entry<String, TaskChannelFactory> entry : taskPluginManager.getTaskChannelFactoryMap().entrySet()) {
|
||||
TaskPluginManager.loadPlugin();
|
||||
for (Map.Entry<String, TaskChannelFactory> entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) {
|
||||
String taskPluginName = entry.getKey();
|
||||
TaskChannelFactory taskChannelFactory = entry.getValue();
|
||||
List<PluginParams> params = taskChannelFactory.getParams();
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditResourceType;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class AuditMessage {
|
||||
|
||||
private User user;
|
||||
|
||||
private Date auditDate;
|
||||
|
||||
private AuditResourceType resourceType;
|
||||
|
||||
private AuditOperationType operation;
|
||||
|
||||
private Integer resourceId;
|
||||
|
||||
public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation,
|
||||
Integer resourceId) {
|
||||
this.user = user;
|
||||
this.auditDate = auditDate;
|
||||
this.resourceType = resourceType;
|
||||
this.operation = operation;
|
||||
this.resourceId = resourceId;
|
||||
}
|
||||
|
||||
public User getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public void setUser(User user) {
|
||||
this.user = user;
|
||||
}
|
||||
|
||||
public Date getAuditDate() {
|
||||
return auditDate;
|
||||
}
|
||||
|
||||
public void setAuditDate(Date auditDate) {
|
||||
this.auditDate = auditDate;
|
||||
}
|
||||
|
||||
public AuditResourceType getResourceType() {
|
||||
return resourceType;
|
||||
}
|
||||
|
||||
public void setResourceType(AuditResourceType resourceType) {
|
||||
this.resourceType = resourceType;
|
||||
}
|
||||
|
||||
public AuditOperationType getOperation() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
public void setOperation(AuditOperationType operation) {
|
||||
this.operation = operation;
|
||||
}
|
||||
|
||||
public Integer getResourceId() {
|
||||
return resourceId;
|
||||
}
|
||||
|
||||
public void setResourceId(Integer resourceId) {
|
||||
this.resourceId = resourceId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AuditMessage{"
|
||||
+ "user=" + user
|
||||
+ ", Date=" + auditDate
|
||||
+ ", resourceType" + resourceType
|
||||
+ ", operation=" + operation
|
||||
+ ", resourceId='" + resourceId + '\'';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
|
||||
import org.apache.dolphinscheduler.api.configuration.ApiConfig;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class AuditPublishService {
|
||||
|
||||
private final BlockingQueue<AuditMessage> auditMessageQueue = new LinkedBlockingQueue<>();
|
||||
|
||||
@Autowired
|
||||
private List<AuditSubscriber> subscribers;
|
||||
|
||||
@Autowired
|
||||
private ApiConfig apiConfig;
|
||||
|
||||
/**
|
||||
* create a daemon thread to process the message queue
|
||||
*/
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
if (apiConfig.isAuditEnable()) {
|
||||
Thread thread = new Thread(this::doPublish);
|
||||
thread.setDaemon(true);
|
||||
thread.setName("Audit-Log-Consume-Thread");
|
||||
thread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* publish a new audit message
|
||||
*
|
||||
* @param message audit message
|
||||
*/
|
||||
public void publish(AuditMessage message) {
|
||||
if (apiConfig.isAuditEnable() && !auditMessageQueue.offer(message)) {
|
||||
log.error("Publish audit message failed, message:{}", message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* subscribers execute the message processor method
|
||||
*/
|
||||
private void doPublish() {
|
||||
AuditMessage message = null;
|
||||
while (true) {
|
||||
try {
|
||||
message = auditMessageQueue.take();
|
||||
for (AuditSubscriber subscriber : subscribers) {
|
||||
try {
|
||||
subscriber.execute(message);
|
||||
} catch (Exception e) {
|
||||
log.error("Consume audit message failed, message:{}", message, e);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Consume audit message failed, message:{}", message, e);
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
/**
|
||||
* Custom annotation for logging and auditing operator actions in the system.
|
||||
* This annotation can be applied to methods to indicate the type of operation, object type,
|
||||
* and specific parameters to be recorded in the logs.
|
||||
*/
|
||||
@Target(ElementType.METHOD)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Documented
|
||||
public @interface OperatorLog {
|
||||
|
||||
AuditType auditType();
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.AuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.aspectj.lang.JoinPoint;
|
||||
import org.aspectj.lang.annotation.AfterReturning;
|
||||
import org.aspectj.lang.annotation.AfterThrowing;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.aspectj.lang.annotation.Before;
|
||||
import org.aspectj.lang.annotation.Pointcut;
|
||||
import org.aspectj.lang.reflect.MethodSignature;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
@Aspect
|
||||
@Slf4j
|
||||
@Component
|
||||
public class OperatorLogAspect {
|
||||
|
||||
private static final ThreadLocal<AuditContext> auditThreadLocal = new ThreadLocal<>();
|
||||
|
||||
@Pointcut("@annotation(org.apache.dolphinscheduler.api.audit.OperatorLog)")
|
||||
public void logPointCut() {
|
||||
}
|
||||
|
||||
@Before("logPointCut()")
|
||||
public void before(JoinPoint point) {
|
||||
MethodSignature signature = (MethodSignature) point.getSignature();
|
||||
Method method = signature.getMethod();
|
||||
OperatorLog operatorLog = method.getAnnotation(OperatorLog.class);
|
||||
Operation operation = method.getAnnotation(Operation.class);
|
||||
|
||||
if (operation == null) {
|
||||
log.warn("Operation is null of method: {}", method.getName());
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> paramsMap = OperatorUtils.getParamsMap(point, signature);
|
||||
User user = OperatorUtils.getUser(paramsMap);
|
||||
if (user == null) {
|
||||
log.error("user is null");
|
||||
return;
|
||||
}
|
||||
|
||||
AuditType auditType = operatorLog.auditType();
|
||||
|
||||
try {
|
||||
AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass());
|
||||
List<AuditLog> auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user);
|
||||
operator.setRequestParam(auditType, auditLogList, paramsMap);
|
||||
AuditContext auditContext =
|
||||
new AuditContext(auditLogList, paramsMap, operatorLog, System.currentTimeMillis(), operator);
|
||||
auditThreadLocal.set(auditContext);
|
||||
} catch (Throwable throwable) {
|
||||
log.error("Record audit log error", throwable);
|
||||
}
|
||||
}
|
||||
|
||||
@AfterReturning(value = "logPointCut()", returning = "returnValue")
|
||||
public void afterReturning(Object returnValue) {
|
||||
try {
|
||||
AuditContext auditContext = auditThreadLocal.get();
|
||||
if (auditContext == null) {
|
||||
return;
|
||||
}
|
||||
auditContext.getOperator().recordAudit(auditContext, returnValue);
|
||||
} catch (Throwable throwable) {
|
||||
log.error("Record audit log error", throwable);
|
||||
} finally {
|
||||
auditThreadLocal.remove();
|
||||
}
|
||||
}
|
||||
|
||||
@AfterThrowing("logPointCut()")
|
||||
public void afterThrowing() {
|
||||
auditThreadLocal.remove();
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@AllArgsConstructor
|
||||
public static class AuditContext {
|
||||
|
||||
List<AuditLog> auditLogList;
|
||||
Map<String, Object> paramsMap;
|
||||
OperatorLog operatorLog;
|
||||
long beginTime;
|
||||
AuditOperator operator;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.enums.ExecuteType;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditModelType;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.aspectj.lang.JoinPoint;
|
||||
import org.aspectj.lang.reflect.MethodSignature;
|
||||
|
||||
@Slf4j
|
||||
public class OperatorUtils {
|
||||
|
||||
public static boolean resultFail(Result<?> result) {
|
||||
return result != null && result.isFailed();
|
||||
}
|
||||
|
||||
public static List<AuditLog> buildAuditLogList(String apiDescription, AuditType auditType, User user) {
|
||||
List<AuditLog> auditLogList = new ArrayList<>();
|
||||
AuditLog auditLog = new AuditLog();
|
||||
auditLog.setUserId(user.getId());
|
||||
auditLog.setModelType(auditType.getAuditModelType().getName());
|
||||
auditLog.setOperationType(auditType.getAuditOperationType().getName());
|
||||
auditLog.setDescription(apiDescription);
|
||||
auditLog.setCreateTime(new Date());
|
||||
auditLogList.add(auditLog);
|
||||
return auditLogList;
|
||||
}
|
||||
|
||||
public static User getUser(Map<String, Object> paramsMap) {
|
||||
for (Object object : paramsMap.values()) {
|
||||
if (object instanceof User) {
|
||||
return (User) object;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Map<String, Object> getParamsMap(JoinPoint point, MethodSignature signature) {
|
||||
Object[] args = point.getArgs();
|
||||
String[] strings = signature.getParameterNames();
|
||||
|
||||
Map<String, Object> paramsMap = new HashMap<>();
|
||||
for (int i = 0; i < strings.length; i++) {
|
||||
paramsMap.put(strings[i], args[i]);
|
||||
}
|
||||
|
||||
return paramsMap;
|
||||
}
|
||||
|
||||
public static AuditOperationType modifyReleaseOperationType(AuditType auditType, Map<String, Object> paramsMap) {
|
||||
switch (auditType.getAuditOperationType()) {
|
||||
case RELEASE:
|
||||
ReleaseState releaseState = (ReleaseState) paramsMap.get(Constants.RELEASE_STATE);
|
||||
if (releaseState == null) {
|
||||
break;
|
||||
}
|
||||
switch (releaseState) {
|
||||
case ONLINE:
|
||||
return AuditOperationType.ONLINE;
|
||||
case OFFLINE:
|
||||
return AuditOperationType.OFFLINE;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case EXECUTE:
|
||||
ExecuteType executeType = (ExecuteType) paramsMap.get(Constants.EXECUTE_TYPE);
|
||||
if (executeType == null) {
|
||||
break;
|
||||
}
|
||||
switch (executeType) {
|
||||
case REPEAT_RUNNING:
|
||||
return AuditOperationType.RERUN;
|
||||
case RECOVER_SUSPENDED_PROCESS:
|
||||
return AuditOperationType.RESUME_PAUSE;
|
||||
case START_FAILURE_TASK_PROCESS:
|
||||
return AuditOperationType.RESUME_FAILURE;
|
||||
case STOP:
|
||||
return AuditOperationType.STOP;
|
||||
case PAUSE:
|
||||
return AuditOperationType.PAUSE;
|
||||
case EXECUTE_TASK:
|
||||
return AuditOperationType.EXECUTE;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return auditType.getAuditOperationType();
|
||||
}
|
||||
|
||||
public static long getObjectIdentityByParma(String[] paramNameArr, Map<String, Object> paramsMap) {
|
||||
for (String name : paramNameArr) {
|
||||
if (paramsMap.get(name) instanceof String) {
|
||||
String param = (String) paramsMap.get(name);
|
||||
try {
|
||||
if (param.matches("\\d+")) {
|
||||
return Long.parseLong(param);
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
public static Map<String, Object> getObjectIfFromReturnObject(Object obj, String[] params) {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
|
||||
try {
|
||||
Class<?> clazz = obj.getClass();
|
||||
|
||||
if (clazz.equals(Long.class)) {
|
||||
map.put(params[0], obj);
|
||||
}
|
||||
|
||||
while (clazz != null) {
|
||||
Field[] fields = clazz.getDeclaredFields();
|
||||
for (Field field : fields) {
|
||||
field.setAccessible(true);
|
||||
|
||||
if (field.getName().equals(params[0])) {
|
||||
map.put(params[0], field.get(obj));
|
||||
}
|
||||
}
|
||||
|
||||
clazz = clazz.getSuperclass();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("get object if from return object error", e);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
public static boolean isUdfResource(Map<String, Object> paramsMap) {
|
||||
ResourceType resourceType = (ResourceType) paramsMap.get(Constants.STRING_PLUGIN_PARAM_TYPE);
|
||||
return resourceType != null && resourceType.equals(ResourceType.UDF);
|
||||
}
|
||||
|
||||
public static boolean isFolder(String name) {
|
||||
return name != null && name.endsWith("/");
|
||||
}
|
||||
|
||||
public static String getFileAuditObject(AuditType auditType, Map<String, Object> paramsMap, String name) {
|
||||
boolean isUdfResource = isUdfResource(paramsMap);
|
||||
boolean isFolder = auditType == AuditType.FOLDER_CREATE || isFolder(name);
|
||||
if (isUdfResource) {
|
||||
return isFolder ? AuditModelType.UDF_FOLDER.getName() : AuditModelType.UDF_FILE.getName();
|
||||
} else {
|
||||
return isFolder ? AuditModelType.FOLDER.getName() : AuditModelType.FILE.getName();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.constants;
|
||||
|
||||
public final class AuditLogConstants {
|
||||
|
||||
private AuditLogConstants() {
|
||||
throw new UnsupportedOperationException("Construct Constants");
|
||||
}
|
||||
|
||||
public static final String CODE = "code";
|
||||
public static final String CODES = "codes";
|
||||
public static final String VERSION = "version";
|
||||
public static final String PROCESS_DEFINITION_CODE = "processDefinitionCode";
|
||||
public static final String PROCESS_DEFINITION_CODES = "processDefinitionCodes";
|
||||
public static final String PROCESS_INSTANCE_IDS = "processInstanceIds";
|
||||
public static final String PROCESS_INSTANCE_ID = "processInstanceId";
|
||||
public static final String WORKFLOW_DEFINITION_CODE = "workflowDefinitionCode";
|
||||
public static final String TYPE = "type";
|
||||
public static final String NAME = "name";
|
||||
public static final String ID = "id";
|
||||
public static final String USER_ID = "userId";
|
||||
public static final String QUEUE_ID = "queueId";
|
||||
public static final String PRIORITY = "priority";
|
||||
public static final String CLUSTER_CODE = "clusterCode";
|
||||
public static final String ENVIRONMENT_CODE = "environmentCode";
|
||||
public static final String ALIAS = "alias";
|
||||
public static final String FILE_NAME = "fileName";
|
||||
public static final String FULL_NAME = "fullName";
|
||||
public static final String FUNC_NAME = "funcName";
|
||||
public static final String UDF_FUNC_ID = "udfFuncId";
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,252 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.enums;
|
||||
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ALIAS;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CLUSTER_CODE;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CODE;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CODES;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ENVIRONMENT_CODE;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FILE_NAME;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FULL_NAME;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FUNC_NAME;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ID;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.NAME;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PRIORITY;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_DEFINITION_CODE;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_DEFINITION_CODES;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_INSTANCE_ID;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_INSTANCE_IDS;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.QUEUE_ID;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.TYPE;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.UDF_FUNC_ID;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.USER_ID;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.VERSION;
|
||||
import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.WORKFLOW_DEFINITION_CODE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.ALARM_GROUP;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.ALARM_INSTANCE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.CLUSTER;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.DATASOURCE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.ENVIRONMENT;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.FILE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.FOLDER;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.K8S_NAMESPACE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROCESS;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROCESS_INSTANCE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROJECT;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.SCHEDULE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK_GROUP;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK_INSTANCE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.TENANT;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.TOKEN;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.UDF_FUNCTION;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.USER;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.WORKER_GROUP;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditModelType.YARN_QUEUE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_DELETE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_RERUN;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_START;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.CLOSE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.COPY;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.CREATE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.DELETE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.DELETE_VERSION;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.EXECUTE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.EXPORT;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.FORCE_SUCCESS;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.IMPORT;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.MODIFY;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.OFFLINE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.ONLINE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.RELEASE;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.START;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.SWITCH_VERSION;
|
||||
import static org.apache.dolphinscheduler.common.enums.AuditOperationType.UPDATE;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.AuditOperator;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.AlertGroupAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.AlertInstanceAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ClusterAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.DatasourceAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.EnvironmentAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.K8SNamespaceAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ProcessAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ProcessInstanceAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ProjectAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ResourceAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.ScheduleAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.TaskAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.TaskGroupAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.TaskInstancesAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.TenantAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.TokenAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.UdfFunctionAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.UserAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.WorkerGroupAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.impl.YarnQueueAuditOperatorImpl;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditModelType;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public enum AuditType {
|
||||
|
||||
PROJECT_CREATE(PROJECT, CREATE, ProjectAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
PROJECT_UPDATE(PROJECT, UPDATE, ProjectAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
PROJECT_DELETE(PROJECT, DELETE, ProjectAuditOperatorImpl.class, new String[]{CODE}, new String[]{}),
|
||||
|
||||
PROCESS_CREATE(PROCESS, CREATE, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
PROCESS_UPDATE(PROCESS, UPDATE, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
PROCESS_SWITCH_VERSION(PROCESS, SWITCH_VERSION, ProcessAuditOperatorImpl.class, new String[]{CODE, VERSION},
|
||||
new String[]{}),
|
||||
PROCESS_DELETE_VERSION(PROCESS, DELETE_VERSION, ProcessAuditOperatorImpl.class, new String[]{CODE, VERSION},
|
||||
new String[]{}),
|
||||
PROCESS_RELEASE(PROCESS, RELEASE, ProcessAuditOperatorImpl.class, new String[]{WORKFLOW_DEFINITION_CODE},
|
||||
new String[]{}),
|
||||
PROCESS_COPY(PROCESS, COPY, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}),
|
||||
PROCESS_EXPORT(PROCESS, EXPORT, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}),
|
||||
PROCESS_DELETE(PROCESS, DELETE, ProcessAuditOperatorImpl.class, new String[]{CODE}, new String[]{}),
|
||||
PROCESS_BATCH_DELETE(PROCESS, BATCH_DELETE, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}),
|
||||
PROCESS_START(PROCESS, START, ProcessAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODE},
|
||||
new String[]{}),
|
||||
PROCESS_BATCH_START(PROCESS, BATCH_START, ProcessAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODES},
|
||||
new String[]{}),
|
||||
PROCESS_BATCH_RERUN(PROCESS, BATCH_RERUN, ProcessInstanceAuditOperatorImpl.class,
|
||||
new String[]{PROCESS_INSTANCE_IDS},
|
||||
new String[]{}),
|
||||
PROCESS_EXECUTE(PROCESS, EXECUTE, ProcessInstanceAuditOperatorImpl.class, new String[]{PROCESS_INSTANCE_ID},
|
||||
new String[]{}),
|
||||
PROCESS_IMPORT(PROCESS, IMPORT, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
PROCESS_INSTANCE_UPDATE(PROCESS_INSTANCE, UPDATE, ProcessInstanceAuditOperatorImpl.class, new String[]{ID},
|
||||
new String[]{}),
|
||||
PROCESS_INSTANCE_DELETE(PROCESS_INSTANCE, DELETE, ProcessInstanceAuditOperatorImpl.class, new String[]{ID},
|
||||
new String[]{}),
|
||||
PROCESS_INSTANCE_BATCH_DELETE(PROCESS_INSTANCE, BATCH_DELETE, ProcessInstanceAuditOperatorImpl.class,
|
||||
new String[]{PROCESS_INSTANCE_IDS}, new String[]{}),
|
||||
|
||||
TASK_CREATE(TASK, CREATE, TaskAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
TASK_UPDATE(TASK, UPDATE, TaskAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
TASK_SWITCH_VERSION(TASK, SWITCH_VERSION, TaskAuditOperatorImpl.class, new String[]{CODE, VERSION}, new String[]{}),
|
||||
TASK_DELETE_VERSION(TASK, DELETE_VERSION, TaskAuditOperatorImpl.class, new String[]{CODE, VERSION}, new String[]{}),
|
||||
TASK_DELETE(TASK, DELETE, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}),
|
||||
TASK_RELEASE(TASK, RELEASE, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}),
|
||||
TASK_START(TASK, START, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}),
|
||||
TASK_INSTANCE_FORCE_SUCCESS(TASK_INSTANCE, FORCE_SUCCESS, TaskInstancesAuditOperatorImpl.class, new String[]{ID},
|
||||
new String[]{}),
|
||||
|
||||
SCHEDULE_CREATE(SCHEDULE, CREATE, ScheduleAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODE},
|
||||
new String[]{ID}),
|
||||
SCHEDULE_UPDATE(SCHEDULE, UPDATE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
SCHEDULE_ONLINE(SCHEDULE, ONLINE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
SCHEDULE_OFFLINE(SCHEDULE, OFFLINE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
SCHEDULE_DELETE(SCHEDULE, DELETE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
FOLDER_CREATE(FOLDER, CREATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, ALIAS}, new String[]{}),
|
||||
FILE_CREATE(FILE, CREATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, FILE_NAME, ALIAS}, new String[]{}),
|
||||
FILE_UPDATE(FILE, UPDATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, FILE_NAME, ALIAS}, new String[]{}),
|
||||
FILE_DELETE(FILE, DELETE, ResourceAuditOperatorImpl.class, new String[]{FULL_NAME}, new String[]{}),
|
||||
|
||||
UDF_FUNCTION_CREATE(UDF_FUNCTION, CREATE, UdfFunctionAuditOperatorImpl.class, new String[]{FUNC_NAME},
|
||||
new String[]{}),
|
||||
UDF_FUNCTION_UPDATE(UDF_FUNCTION, UPDATE, UdfFunctionAuditOperatorImpl.class, new String[]{FUNC_NAME},
|
||||
new String[]{}),
|
||||
UDF_FUNCTION_DELETE(UDF_FUNCTION, DELETE, UdfFunctionAuditOperatorImpl.class, new String[]{UDF_FUNC_ID},
|
||||
new String[]{}),
|
||||
|
||||
TASK_GROUP_CREATE(TASK_GROUP, CREATE, TaskGroupAuditOperatorImpl.class, new String[]{NAME}, new String[]{}),
|
||||
TASK_GROUP_UPDATE(TASK_GROUP, UPDATE, TaskGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
TASK_GROUP_CLOSE(TASK_GROUP, CLOSE, TaskGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
TASK_GROUP_START(TASK_GROUP, START, TaskGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
TASK_GROUP_MODIFY(TASK_GROUP, MODIFY, TaskGroupAuditOperatorImpl.class, new String[]{QUEUE_ID, PRIORITY},
|
||||
new String[]{}),
|
||||
|
||||
DATASOURCE_CREATE(DATASOURCE, CREATE, DatasourceAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
DATASOURCE_UPDATE(DATASOURCE, UPDATE, DatasourceAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
DATASOURCE_DELETE(DATASOURCE, DELETE, DatasourceAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
TENANT_CREATE(TENANT, CREATE, TenantAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
TENANT_UPDATE(TENANT, UPDATE, TenantAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
TENANT_DELETE(TENANT, DELETE, TenantAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
USER_CREATE(USER, CREATE, UserAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
USER_UPDATE(USER, UPDATE, UserAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
USER_DELETE(USER, DELETE, UserAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
ALARM_GROUP_CREATE(ALARM_GROUP, CREATE, AlertGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
ALARM_GROUP_UPDATE(ALARM_GROUP, UPDATE, AlertGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
ALARM_GROUP_DELETE(ALARM_GROUP, DELETE, AlertGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
ALARM_INSTANCE_CREATE(ALARM_INSTANCE, CREATE, AlertInstanceAuditOperatorImpl.class, new String[]{},
|
||||
new String[]{ID}),
|
||||
ALARM_INSTANCE_UPDATE(ALARM_INSTANCE, UPDATE, AlertInstanceAuditOperatorImpl.class, new String[]{},
|
||||
new String[]{ID}),
|
||||
ALARM_INSTANCE_DELETE(ALARM_INSTANCE, DELETE, AlertInstanceAuditOperatorImpl.class, new String[]{ID},
|
||||
new String[]{}),
|
||||
|
||||
WORKER_GROUP_CREATE(WORKER_GROUP, CREATE, WorkerGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
WORKER_GROUP_DELETE(WORKER_GROUP, DELETE, WorkerGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
YARN_QUEUE_CREATE(YARN_QUEUE, CREATE, YarnQueueAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
YARN_QUEUE_UPDATE(YARN_QUEUE, UPDATE, YarnQueueAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
YARN_QUEUE_DELETE(YARN_QUEUE, DELETE, YarnQueueAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
ENVIRONMENT_CREATE(ENVIRONMENT, CREATE, EnvironmentAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
ENVIRONMENT_UPDATE(ENVIRONMENT, UPDATE, EnvironmentAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
ENVIRONMENT_DELETE(ENVIRONMENT, DELETE, EnvironmentAuditOperatorImpl.class, new String[]{ENVIRONMENT_CODE},
|
||||
new String[]{}),
|
||||
|
||||
CLUSTER_CREATE(CLUSTER, CREATE, ClusterAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
CLUSTER_UPDATE(CLUSTER, UPDATE, ClusterAuditOperatorImpl.class, new String[]{}, new String[]{CODE}),
|
||||
CLUSTER_DELETE(CLUSTER, DELETE, ClusterAuditOperatorImpl.class, new String[]{CLUSTER_CODE}, new String[]{}),
|
||||
|
||||
K8S_NAMESPACE_CREATE(K8S_NAMESPACE, CREATE, K8SNamespaceAuditOperatorImpl.class, new String[]{}, new String[]{ID}),
|
||||
K8S_NAMESPACE_DELETE(K8S_NAMESPACE, DELETE, K8SNamespaceAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
|
||||
TOKEN_CREATE(TOKEN, CREATE, TokenAuditOperatorImpl.class, new String[]{}, new String[]{USER_ID}),
|
||||
TOKEN_UPDATE(TOKEN, UPDATE, TokenAuditOperatorImpl.class, new String[]{}, new String[]{USER_ID}),
|
||||
TOKEN_DELETE(TOKEN, DELETE, TokenAuditOperatorImpl.class, new String[]{ID}, new String[]{}),
|
||||
;
|
||||
|
||||
private final Class<? extends AuditOperator> operatorClass;
|
||||
private final AuditModelType auditModelType;
|
||||
private final AuditOperationType auditOperationType;
|
||||
|
||||
/**
|
||||
* The names of the fields in the API request to be recorded.
|
||||
* Represents an array of key-value pairs, e.g., [ID, "status"].
|
||||
*/
|
||||
private final String[] requestParamName;
|
||||
|
||||
/**
|
||||
* The names of the fields in the returned object to be recorded.
|
||||
* Represents an array of field names, e.g., [ID, CODE].
|
||||
* Specify the field names to record from the returned object.
|
||||
*/
|
||||
private final String[] returnObjectFieldName;
|
||||
|
||||
AuditType(AuditModelType auditModelType, AuditOperationType auditOperationType,
|
||||
Class<? extends AuditOperator> operatorClass, String[] requestParamName, String[] returnObjectFieldName) {
|
||||
this.auditModelType = auditModelType;
|
||||
this.auditOperationType = auditOperationType;
|
||||
this.operatorClass = operatorClass;
|
||||
this.requestParamName = requestParamName;
|
||||
this.returnObjectFieldName = returnObjectFieldName;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLogAspect;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface AuditOperator {
|
||||
|
||||
void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue);
|
||||
|
||||
void setRequestParam(AuditType auditType, List<AuditLog> auditLogList, Map<String, Object> paramsMap);
|
||||
}
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLog;
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLogAspect;
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.service.AuditService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public abstract class BaseAuditOperator implements AuditOperator {
|
||||
|
||||
@Autowired
|
||||
private AuditService auditService;
|
||||
|
||||
@Override
|
||||
public void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue) {
|
||||
Result<?> result = new Result<>();
|
||||
|
||||
if (returnValue instanceof Result) {
|
||||
result = (Result<?>) returnValue;
|
||||
if (OperatorUtils.resultFail(result)) {
|
||||
log.error("request fail, code {}", result.getCode());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
long latency = System.currentTimeMillis() - auditContext.getBeginTime();
|
||||
List<AuditLog> auditLogList = auditContext.getAuditLogList();
|
||||
|
||||
Map<String, Object> paramsMap = auditContext.getParamsMap();
|
||||
OperatorLog operatorLog = auditContext.getOperatorLog();
|
||||
AuditType auditType = operatorLog.auditType();
|
||||
|
||||
setObjectIdentityFromReturnObject(auditType, result, auditLogList);
|
||||
modifyAuditOperationType(auditType, paramsMap, auditLogList);
|
||||
modifyAuditObjectType(auditType, paramsMap, auditLogList);
|
||||
|
||||
auditLogList.forEach(auditLog -> auditLog.setLatency(latency));
|
||||
auditLogList.forEach(auditLog -> auditService.addAudit(auditLog));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRequestParam(AuditType auditType, List<AuditLog> auditLogList, Map<String, Object> paramsMap) {
|
||||
String[] paramNameArr = auditType.getRequestParamName();
|
||||
|
||||
if (paramNameArr.length == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
modifyRequestParams(paramNameArr, paramsMap, auditLogList);
|
||||
setObjectByParma(paramNameArr, paramsMap, auditLogList);
|
||||
|
||||
if (auditLogList.get(0).getModelId() == null) {
|
||||
auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParma(paramNameArr, paramsMap));
|
||||
}
|
||||
}
|
||||
|
||||
protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
String name = paramNameArr[0];
|
||||
Object value = paramsMap.get(name);
|
||||
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
String objName = getObjectNameFromReturnIdentity(value);
|
||||
|
||||
if (Strings.isNullOrEmpty(objName)) {
|
||||
auditLogList.get(0).setModelName(value.toString());
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
long objectId = Long.parseLong(value.toString());
|
||||
auditLogList.get(0).setModelId(objectId);
|
||||
} catch (NumberFormatException e) {
|
||||
log.error("value is not long, value: {}", value);
|
||||
}
|
||||
|
||||
auditLogList.get(0).setModelName(objName);
|
||||
}
|
||||
|
||||
protected void setObjectByParmaArr(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
AuditLog auditLog = auditLogList.get(0);
|
||||
for (String param : paramNameArr) {
|
||||
if (!paramsMap.containsKey(param)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String[] identityArr = ((String) paramsMap.get(param)).split(",");
|
||||
for (String identityString : identityArr) {
|
||||
long identity = toLong(identityString);
|
||||
|
||||
String value = getObjectNameFromReturnIdentity(identity);
|
||||
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auditLog.setModelId(identity);
|
||||
auditLog.setModelName(value);
|
||||
auditLogList.add(auditLog);
|
||||
auditLog = AuditLog.copyNewOne(auditLog);
|
||||
}
|
||||
}
|
||||
auditLogList.remove(0);
|
||||
}
|
||||
|
||||
protected void setObjectIdentityFromReturnObject(AuditType auditType, Result<?> result,
|
||||
List<AuditLog> auditLogList) {
|
||||
String[] returnObjectFieldNameArr = auditType.getReturnObjectFieldName();
|
||||
if (returnObjectFieldNameArr.length == 0) {
|
||||
return;
|
||||
}
|
||||
Map<String, Object> returnObjectMap =
|
||||
OperatorUtils.getObjectIfFromReturnObject(result.getData(), returnObjectFieldNameArr);
|
||||
modifyObjectFromReturnObject(returnObjectFieldNameArr, returnObjectMap, auditLogList);
|
||||
setObjectNameFromReturnIdentity(auditLogList);
|
||||
}
|
||||
|
||||
protected void setObjectNameFromReturnIdentity(List<AuditLog> auditLogList) {
|
||||
auditLogList
|
||||
.forEach(auditLog -> auditLog.setModelName(getObjectNameFromReturnIdentity(auditLog.getModelId())));
|
||||
}
|
||||
|
||||
protected void modifyObjectFromReturnObject(String[] params, Map<String, Object> returnObjectMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
if (returnObjectMap.isEmpty() || returnObjectMap.get(params[0]) == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Long objId = toLong(returnObjectMap.get(params[0]));
|
||||
|
||||
if (objId != -1) {
|
||||
auditLogList.get(0).setModelId(objId);
|
||||
}
|
||||
}
|
||||
|
||||
protected Long toLong(Object str) {
|
||||
if (str == null) {
|
||||
return -1L;
|
||||
}
|
||||
|
||||
return NumberUtils.toLong(str.toString(), -1);
|
||||
}
|
||||
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
return identity.toString();
|
||||
}
|
||||
|
||||
protected void modifyRequestParams(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
}
|
||||
|
||||
protected void modifyAuditObjectType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
}
|
||||
|
||||
protected void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -15,28 +15,29 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit;
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
|
||||
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Component
|
||||
public class AuditSubscriberImpl implements AuditSubscriber {
|
||||
@Service
|
||||
public class AlertGroupAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private AuditLogMapper logMapper;
|
||||
private AlertGroupMapper alertGroupMapper;
|
||||
|
||||
@Override
|
||||
public void execute(AuditMessage message) {
|
||||
AuditLog auditLog = new AuditLog();
|
||||
auditLog.setUserId(message.getUser().getId());
|
||||
auditLog.setResourceType(message.getResourceType().getCode());
|
||||
auditLog.setOperation(message.getOperation().getCode());
|
||||
auditLog.setTime(message.getAuditDate());
|
||||
auditLog.setResourceId(message.getResourceId());
|
||||
logMapper.insert(auditLog);
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
AlertGroup obj = alertGroupMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getGroupName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
|
||||
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class AlertInstanceAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private AlertPluginInstanceMapper alertPluginInstanceMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
AlertPluginInstance obj = alertPluginInstanceMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getInstanceName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.Cluster;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class ClusterAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private ClusterMapper clusterMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Cluster obj = clusterMapper.queryByClusterCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.DataSource;
|
||||
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class DatasourceAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private DataSourceMapper dataSourceMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
DataSource obj = dataSourceMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.Environment;
|
||||
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class EnvironmentAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private EnvironmentMapper environmentMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Environment obj = environmentMapper.queryByEnvironmentCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.K8sNamespace;
|
||||
import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class K8SNamespaceAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private K8sNamespaceMapper k8sNamespaceMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
K8sNamespace obj = k8sNamespaceMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getNamespace();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class ProcessAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private ProcessDefinitionMapper processDefinitionMapper;
|
||||
|
||||
@Override
|
||||
public void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap);
|
||||
auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
if (paramNameArr[0].equals(AuditLogConstants.CODES)
|
||||
|| paramNameArr[0].equals(AuditLogConstants.PROCESS_DEFINITION_CODES)
|
||||
|| paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) {
|
||||
super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList);
|
||||
} else {
|
||||
super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
|
||||
}
|
||||
if (paramsMap.containsKey(AuditLogConstants.VERSION)) {
|
||||
if (paramsMap.get(AuditLogConstants.VERSION) != null) {
|
||||
auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString());
|
||||
} else {
|
||||
auditLogList.get(0).setDetail("latest");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
ProcessDefinition obj = processDefinitionMapper.queryByCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class ProcessInstanceAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private ProcessInstanceMapper processInstanceMapper;
|
||||
|
||||
@Override
|
||||
public void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap);
|
||||
auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
if (paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) {
|
||||
super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList);
|
||||
} else {
|
||||
super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
int objId = NumberUtils.toInt(identity.toString(), -1);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
ProcessInstance obj = processInstanceMapper.queryDetailById(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ProjectAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private ProjectMapper projectMapper;
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Project obj = projectMapper.queryByCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class ResourceAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Override
|
||||
public void modifyAuditObjectType(AuditType auditType, Map<String, Object> paramsMap, List<AuditLog> auditLogList) {
|
||||
auditLogList.forEach(auditLog -> auditLog
|
||||
.setModelType(OperatorUtils.getFileAuditObject(auditType, paramsMap, auditLog.getModelName())));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
Object objName = getFileNameFromParam(paramNameArr, paramsMap);
|
||||
|
||||
if (objName == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
auditLogList.get(0).setModelName(objName.toString());
|
||||
}
|
||||
|
||||
private String getFileNameFromParam(String[] paramNameArr, Map<String, Object> paramsMap) {
|
||||
for (String param : paramNameArr) {
|
||||
if (!param.equals("type") && paramsMap.containsKey(param)) {
|
||||
return paramsMap.get(param).toString();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.Schedule;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class ScheduleAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private ScheduleMapper scheduleMapper;
|
||||
|
||||
@Autowired
|
||||
private ProcessDefinitionMapper processDefinitionMapper;
|
||||
|
||||
@Override
|
||||
public void modifyRequestParams(String[] paramNameArr, Map<String, Object> paramsMap, List<AuditLog> auditLogList) {
|
||||
if (!paramNameArr[0].equals(AuditLogConstants.ID)) {
|
||||
return;
|
||||
}
|
||||
int id = (int) paramsMap.get(paramNameArr[0]);
|
||||
Schedule schedule = scheduleMapper.selectById(id);
|
||||
if (schedule != null) {
|
||||
paramsMap.put(AuditLogConstants.CODE, schedule.getProcessDefinitionCode());
|
||||
paramNameArr[0] = AuditLogConstants.CODE;
|
||||
auditLogList.forEach(auditLog -> auditLog.setDetail(String.valueOf(id)));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setObjectIdentityFromReturnObject(AuditType auditType, Result<?> result,
|
||||
List<AuditLog> auditLogList) {
|
||||
String[] returnObjectFieldNameArr = auditType.getReturnObjectFieldName();
|
||||
if (returnObjectFieldNameArr.length == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> returnObjectMap =
|
||||
OperatorUtils.getObjectIfFromReturnObject(result.getData(), returnObjectFieldNameArr);
|
||||
auditLogList
|
||||
.forEach(auditLog -> auditLog.setDetail(returnObjectMap.get(returnObjectFieldNameArr[0]).toString()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
ProcessDefinition obj = processDefinitionMapper.queryByCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorUtils;
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class TaskAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private TaskDefinitionMapper taskDefinitionMapper;
|
||||
|
||||
@Override
|
||||
public void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap);
|
||||
auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setObjectByParma(String[] paramNameArr, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
|
||||
super.setObjectByParma(paramNameArr, paramsMap, auditLogList);
|
||||
if (paramsMap.containsKey(AuditLogConstants.VERSION)) {
|
||||
auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
TaskDefinition obj = taskDefinitionMapper.queryByCode(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskGroup;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class TaskGroupAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private TaskGroupMapper taskGroupMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
TaskGroup obj = taskGroupMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class TaskInstancesAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private TaskInstanceMapper taskInstanceMapper;
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
TaskInstance obj = taskInstanceMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.Tenant;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class TenantAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private TenantMapper tenantMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Tenant obj = tenantMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getTenantCode();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.AccessToken;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class TokenAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private AccessTokenMapper accessTokenMapper;
|
||||
|
||||
@Autowired
|
||||
private UserMapper userMapper;
|
||||
|
||||
@Override
|
||||
public void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
if (paramsMap.get(AuditLogConstants.USER_ID) != null) {
|
||||
User user = userMapper.selectById(paramsMap.get(AuditLogConstants.USER_ID).toString());
|
||||
auditLogList.forEach(auditLog -> {
|
||||
auditLog.setModelName(user.getUserName());
|
||||
auditLog.setModelId(Long.valueOf(user.getId()));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
AccessToken obj = accessTokenMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getUserName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class UdfFunctionAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private UdfFuncMapper udfFuncMapper;
|
||||
|
||||
@Override
|
||||
protected String getObjectNameFromReturnIdentity(Object identity) {
|
||||
int objId = NumberUtils.toInt(identity.toString(), -1);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
UdfFunc obj = udfFuncMapper.selectUdfById(objId);
|
||||
return obj == null ? "" : obj.getFuncName();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class UserAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private UserMapper userMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
User obj = userMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getUserName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.common.enums.AuditOperationType;
|
||||
import org.apache.dolphinscheduler.dao.entity.AuditLog;
|
||||
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
|
||||
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class WorkerGroupAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private WorkerGroupMapper workerGroupMapper;
|
||||
|
||||
@Override
|
||||
public void modifyAuditOperationType(AuditType auditType, Map<String, Object> paramsMap,
|
||||
List<AuditLog> auditLogList) {
|
||||
if (auditType.getAuditOperationType() == AuditOperationType.CREATE
|
||||
&& paramsMap.get(AuditLogConstants.ID) != null &&
|
||||
!paramsMap.get(AuditLogConstants.ID).toString().equals("0")) {
|
||||
auditLogList.forEach(auditLog -> auditLog.setOperationType(AuditOperationType.UPDATE.getName()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
WorkerGroup obj = workerGroupMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getName();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.audit.operator.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator;
|
||||
import org.apache.dolphinscheduler.dao.entity.Queue;
|
||||
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class YarnQueueAuditOperatorImpl extends BaseAuditOperator {
|
||||
|
||||
@Autowired
|
||||
private QueueMapper queueMapper;
|
||||
|
||||
@Override
|
||||
public String getObjectNameFromReturnIdentity(Object identity) {
|
||||
Long objId = toLong(identity);
|
||||
if (objId == -1) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Queue obj = queueMapper.selectById(objId);
|
||||
return obj == null ? "" : obj.getQueueName();
|
||||
}
|
||||
}
|
||||
|
|
@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_BY_
|
|||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_LIST_PAGING_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ACCESS_TOKEN_ERROR;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLog;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
||||
import org.apache.dolphinscheduler.api.service.AccessTokenService;
|
||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
|
|
@ -83,6 +85,7 @@ public class AccessTokenController extends BaseController {
|
|||
@PostMapping()
|
||||
@ResponseStatus(HttpStatus.CREATED)
|
||||
@ApiException(CREATE_ACCESS_TOKEN_ERROR)
|
||||
@OperatorLog(auditType = AuditType.TOKEN_CREATE)
|
||||
public Result<AccessToken> createToken(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@RequestParam(value = "userId") int userId,
|
||||
@RequestParam(value = "expireTime") String expireTime,
|
||||
|
|
@ -169,13 +172,15 @@ public class AccessTokenController extends BaseController {
|
|||
* @return delete result code
|
||||
*/
|
||||
@Parameter(hidden = true)
|
||||
@Operation(summary = "deleteToken", description = "DELETE_TOKEN_NOTES")
|
||||
@DeleteMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(DELETE_ACCESS_TOKEN_ERROR)
|
||||
@OperatorLog(auditType = AuditType.TOKEN_DELETE)
|
||||
public Result<Boolean> delAccessTokenById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id) {
|
||||
accessTokenService.deleteAccessTokenById(loginUser, id);
|
||||
return Result.success(true);
|
||||
return Result.success(false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -198,6 +203,7 @@ public class AccessTokenController extends BaseController {
|
|||
@PutMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(UPDATE_ACCESS_TOKEN_ERROR)
|
||||
@OperatorLog(auditType = AuditType.TOKEN_UPDATE)
|
||||
public Result<AccessToken> updateToken(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id,
|
||||
@RequestParam(value = "userId") int userId,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALERT_GROUP_ERR
|
|||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALL_ALERTGROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ALERT_GROUP_ERROR;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLog;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
||||
import org.apache.dolphinscheduler.api.service.AlertGroupService;
|
||||
|
|
@ -86,6 +88,7 @@ public class AlertGroupController extends BaseController {
|
|||
@PostMapping()
|
||||
@ResponseStatus(HttpStatus.CREATED)
|
||||
@ApiException(CREATE_ALERT_GROUP_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_GROUP_CREATE)
|
||||
public Result<AlertGroup> createAlertGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@RequestParam(value = "groupName") String groupName,
|
||||
@RequestParam(value = "description", required = false) String description,
|
||||
|
|
@ -195,6 +198,7 @@ public class AlertGroupController extends BaseController {
|
|||
@PutMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(UPDATE_ALERT_GROUP_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_GROUP_UPDATE)
|
||||
public Result<AlertGroup> updateAlertGroupById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id,
|
||||
@RequestParam(value = "groupName") String groupName,
|
||||
|
|
@ -219,6 +223,7 @@ public class AlertGroupController extends BaseController {
|
|||
@DeleteMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(DELETE_ALERT_GROUP_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_GROUP_DELETE)
|
||||
public Result<Boolean> deleteAlertGroupById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id) {
|
||||
alertGroupService.deleteAlertGroupById(loginUser, id);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALL_ALERT_PLUGI
|
|||
import static org.apache.dolphinscheduler.api.enums.Status.SEND_TEST_ALERT_PLUGIN_INSTANCE_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ALERT_PLUGIN_INSTANCE_ERROR;
|
||||
|
||||
import org.apache.dolphinscheduler.api.audit.OperatorLog;
|
||||
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
||||
import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService;
|
||||
|
|
@ -91,6 +93,7 @@ public class AlertPluginInstanceController extends BaseController {
|
|||
@PostMapping()
|
||||
@ResponseStatus(HttpStatus.CREATED)
|
||||
@ApiException(CREATE_ALERT_PLUGIN_INSTANCE_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_INSTANCE_CREATE)
|
||||
public Result<AlertPluginInstance> createAlertPluginInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@RequestParam(value = "pluginDefineId") int pluginDefineId,
|
||||
@RequestParam(value = "instanceName") String instanceName,
|
||||
|
|
@ -134,6 +137,7 @@ public class AlertPluginInstanceController extends BaseController {
|
|||
@PutMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(UPDATE_ALERT_PLUGIN_INSTANCE_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_INSTANCE_UPDATE)
|
||||
public Result<AlertPluginInstance> updateAlertPluginInstanceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id,
|
||||
@RequestParam(value = "instanceName") String instanceName,
|
||||
|
|
@ -158,6 +162,7 @@ public class AlertPluginInstanceController extends BaseController {
|
|||
@DeleteMapping(value = "/{id}")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
@ApiException(DELETE_ALERT_PLUGIN_INSTANCE_ERROR)
|
||||
@OperatorLog(auditType = AuditType.ALARM_INSTANCE_DELETE)
|
||||
public Result<Boolean> deleteAlertPluginInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
|
||||
@PathVariable(value = "id") int id) {
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue