diff --git a/.asf.yaml b/.asf.yaml index cf409dc6a3..84619447be 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,4 +49,4 @@ github: - "Mergeable: milestone-label-check" required_pull_request_reviews: dismiss_stale_reviews: true - required_approving_review_count: 1 + required_approving_review_count: 2 diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml new file mode 100644 index 0000000000..f54421b066 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dsip-request.yml @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: DSIP +description: Suggest an idea for this project +title: "[DSIP-][Module Name] DSIP title" +labels: [ "DSIP", "Waiting for reply" ] +body: + - type: markdown + attributes: + value: | + For better global communication, Please write in English. + + If you feel the description in English is not clear, then you can append description in Chinese, thanks! + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first + to see whether the same DSIP was created already. + options: + - label: > + I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no + similar DSIP. + required: true + + - type: textarea + attributes: + label: Motivation + description: Why you want to do this change? + + - type: textarea + attributes: + label: Design Detail + description: Your design. + placeholder: > + It's better to provide a detailed design, such as the design of the interface, the design of the database, etc. + + - type: textarea + attributes: + label: Compatibility, Deprecation, and Migration Plan + description: > + If this feature is related to compatibility, deprecation, or migration, please describe it here. + + - type: textarea + attributes: + label: Test Plan + description: > + How to test this improvement. + + - type: checkboxes + attributes: + label: Code of Conduct + description: | + The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. + options: + - label: | + I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) + required: true + + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ea09a17fd2..671223d286 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -106,10 +106,14 @@ jobs: strategy: matrix: case: - - name: cluster-test-mysql - script: .github/workflows/cluster-test/mysql/start-job.sh - - name: cluster-test-postgresql - script: .github/workflows/cluster-test/postgresql/start-job.sh + - name: cluster-test-mysql-with-zookeeper-registry + script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh + - name: cluster-test-mysql-with-mysql-registry + script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh + - name: cluster-test-postgresql-zookeeper-registry + script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh + - name: cluster-test-postgresql-with-postgresql-registry + script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/cluster-test/mysql/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile similarity index 85% rename from .github/workflows/cluster-test/mysql/Dockerfile rename to .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile index c7d6abe889..12c7db3c18 100644 --- a/.github/workflows/cluster-test/mysql/Dockerfile +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile @@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Download mysql jar ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" @@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER #Deploy -COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh new file mode 100644 index 0000000000..72b2a630fa --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Create database +mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..d59e3c868c --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + mysql: + container_name: mysql + image: mysql:5.7.36 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: 123456 + ports: + - "3306:3306" + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + timeout: 60s + retries: 120 diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh new file mode 100755 index 0000000000..8536eb0905 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export SPRING_DATASOURCE_USERNAME=root +export SPRING_DATASOURCE_PASSWORD=123456 + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export REGISTRY_HIKARI_CONFIG_USERNAME=root +export REGISTRY_HIKARI_CONFIG_PASSWORD=123456 + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql/running_test.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/running_test.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/mysql/start-job.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh similarity index 74% rename from .github/workflows/cluster-test/mysql/start-job.sh rename to .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh index ee67c5179b..0ce48c64ae 100644 --- a/.github/workflows/cluster-test/mysql/start-job.sh +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d #Build ds mysql cluster image -docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql/Dockerfile . +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile . #Start ds mysql cluster container -docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/mysql/running_test.sh +/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile new file mode 100644 index 0000000000..574c059442 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Download mysql jar +ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER + +#Deploy +COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql/deploy.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/deploy.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/mysql/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/mysql/docker-compose-base.yaml rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..7343c8eee7 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_mysql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 97% rename from .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh index 671c70a5bb..f64e59b768 100755 --- a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=123456 # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/.github/workflows/cluster-test/mysql/install_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/mysql/install_env.sh rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..7582c3ccc5 --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh new file mode 100644 index 0000000000..db8d23147e --- /dev/null +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d + +#Build ds mysql cluster image +docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile . + +#Start ds mysql cluster container +docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile new file mode 100644 index 0000000000..bb2d9a5383 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jre + +RUN apt update ; \ + apt install -y wget sudo openssh-server netcat-traditional ; + +COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root +RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ + +RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Deploy +COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh new file mode 100644 index 0000000000..37bf3433c0 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml new file mode 100644 index 0000000000..1793d94f39 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + postgres: + container_name: postgres + image: postgres:14.1 + restart: always + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dolphinscheduler + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh new file mode 100644 index 0000000000..29f8570319 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-postgresql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export SPRING_DATASOURCE_USERNAME=postgres +export SPRING_DATASOURCE_PASSWORD=postgres + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=jdbc +export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" +export REGISTRY_HIKARI_CONFIG_USERNAME=postgres +export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} +export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH + +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + +# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log +#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* +#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH +#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS +#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS +#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh new file mode 100644 index 0000000000..cd660febf8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/running_test.sh rename to .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh new file mode 100644 index 0000000000..e2b6b630e8 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + +#Start base service containers +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d + +#Build ds postgresql cluster image +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile . + +#Start ds postgresql cluster container +docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d + +#Running tests +/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh + +#Cleanup +docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile similarity index 77% rename from .github/workflows/cluster-test/postgresql/Dockerfile rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile index 38234ee7b3..077b5c97b8 100644 --- a/.github/workflows/cluster-test/postgresql/Dockerfile +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile @@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting install.sh -COPY .github/workflows/cluster-test/postgresql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh #Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Deploy -COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh +COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql/deploy.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/deploy.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml similarity index 100% rename from .github/workflows/cluster-test/postgresql/docker-compose-base.yaml rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml new file mode 100644 index 0000000000..9ab79ea44d --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + ds: + container_name: ds + image: jdk8:ds_postgresql_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh similarity index 97% rename from .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh index 1dbd63254e..6851716058 100644 --- a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh @@ -28,7 +28,6 @@ export SPRING_DATASOURCE_PASSWORD=postgres # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh similarity index 100% rename from .github/workflows/cluster-test/postgresql/install_env.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh new file mode 100644 index 0000000000..0bc861c389 --- /dev/null +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" +ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" + +#Cluster start health check +TIMEOUT=180 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi + + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi + +ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") +if [[ $ALERT_HTTP_STATUS -ne 200 ]];then + echo "alert stop health check success" +else + echo "alert stop health check failed" + exit 3 +fi diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh similarity index 73% rename from .github/workflows/cluster-test/postgresql/start-job.sh rename to .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh index ba0878e3ec..fe755c97f1 100644 --- a/.github/workflows/cluster-test/postgresql/start-job.sh +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh @@ -18,16 +18,16 @@ set -euox pipefail #Start base service containers -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d #Build ds postgresql cluster image -docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql/Dockerfile . +docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile . #Start ds postgresql cluster container -docker-compose -f .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d +docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d #Running tests -/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh +/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh #Cleanup docker rm -f $(docker ps -aq) diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 33633f3b2e..ba533b6e47 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -174,7 +174,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | image.master | string | `"dolphinscheduler-master"` | master image | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent | | image.pullSecret | string | `""` | Specify a imagePullSecrets | -| image.registry | string | `"apache/dolphinscheduler"` | Docker image repository for the DolphinScheduler | +| image.registry | string | `"apache"` | Docker image repository for the DolphinScheduler | | image.tag | string | `"latest"` | Docker image version for the DolphinScheduler | | image.tools | string | `"dolphinscheduler-tools"` | tools image | | image.worker | string | `"dolphinscheduler-worker"` | worker image | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 98c2f70db0..7a04ff5604 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -31,7 +31,7 @@ initImage: image: # -- Docker image repository for the DolphinScheduler - registry: apache/dolphinscheduler + registry: apache # -- Docker image version for the DolphinScheduler tag: latest # -- Image pull policy. Options: Always, Never, IfNotPresent diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 13d8932943..fe0b7851ba 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -286,7 +286,6 @@ Location: `master-server/conf/application.yaml` | Parameters | Default value | Description | |-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | master.listen-port | 5678 | master listen port | -| master.fetch-command-num | 10 | the number of commands fetched by master | | master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel | | master.exec-threads | 100 | master execute thread number to limit process instances in parallel | | master.dispatch-task-number | 3 | master dispatch task number per batch | @@ -305,6 +304,9 @@ Location: `master-server/conf/application.yaml` | master.registry-disconnect-strategy.strategy | stop | Used when the master disconnect from registry, default value: stop. Optional values include stop, waiting | | master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely | | master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory | +| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db | +| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master | ### Worker Server related configuration diff --git a/docs/docs/en/contribute/frontend-development.md b/docs/docs/en/contribute/frontend-development.md index 0cc86811f8..50f8c91fb4 100644 --- a/docs/docs/en/contribute/frontend-development.md +++ b/docs/docs/en/contribute/frontend-development.md @@ -33,7 +33,7 @@ Use the command line mode `cd` enter the `dolphinscheduler-ui` project director > If `npm install` is very slow, you can set the taobao mirror ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend: diff --git a/docs/docs/en/faq.md b/docs/docs/en/faq.md index b0954b3468..7ac4afb76e 100644 --- a/docs/docs/en/faq.md +++ b/docs/docs/en/faq.md @@ -459,11 +459,11 @@ A: 1, cd dolphinscheduler-ui and delete node_modules directory sudo rm -rf node_modules ``` -​ 2, install node-sass through npm.taobao.org +​ 2, install node-sass through npmmirror.com ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3, if the 2nd step failure, please, [referer url](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/frontend-development.md) diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index 8e58fdeccd..a6587a5855 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco ## Install DolphinScheduler -Please download the source code package `apache-dolphinscheduler--src.tar.gz`, download address: [download address](https://dolphinscheduler.apache.org/en-us/download) - -To publish the release name `dolphinscheduler` version, please execute the following commands: - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# Choose the corresponding version yourself +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` To publish the release name `dolphinscheduler` version to `test` namespace: diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index e63436f203..7a3b43b00e 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -123,7 +123,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/en/guide/task/spark.md b/docs/docs/en/guide/task/spark.md index 3e0f83b253..930f2cd0b0 100644 --- a/docs/docs/en/guide/task/spark.md +++ b/docs/docs/en/guide/task/spark.md @@ -24,6 +24,7 @@ Spark task type for executing Spark application. When executing the Spark task, |----------------------------|------------------------------------------------------------------------------------------------------------------------------------| | Program type | Supports Java, Scala, Python, and SQL. | | The class of main function | The **full path** of Main Class, the entry point of the Spark program. | +| Master | The The master URL for the cluster. | | Main jar package | The Spark jar package (upload by Resource Center). | | SQL scripts | SQL statements in .sql files that Spark sql runs. | | Deployment mode |
  • spark submit supports three modes: cluster, client and local.
  • spark sql supports client and local modes.
| diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 08fded19e0..d8d1d42d1e 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -281,29 +281,30 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 位置：`master-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -|-----------------------------------------------------------------------------|--------------|-----------------------------------------------------------------------------------------| -| master.listen-port | 5678 | master监听端口 | -| master.fetch-command-num | 10 | master拉取command数量 | -| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | -| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | -| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | -| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | -| master.max-heartbeat-interval | 10s | master最大心跳间隔 | -| master.task-commit-retry-times | 5 | 任务重试次数 | -| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | -| master.state-wheel-interval | 5 | 轮询检查状态时间 | -| master.server-load-protection.enabled | true | 是否开启系统保护策略 | -| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | -| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | -| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | -| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | -| master.failover-interval | 10 | failover间隔，单位为分钟 | -| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | -| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | -| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, | -| 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | -| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| 参数 | 默认值 | 描述 | +|-----------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------| +| master.listen-port | 5678 | master监听端口 | +| master.pre-exec-threads | 10 | master准备执行任务的数量，用于限制并行的command | +| master.exec-threads | 100 | master工作线程数量,用于限制并行的流程实例数量 | +| master.dispatch-task-number | 3 | master每个批次的派发任务数量 | +| master.host-selector | lower_weight | master host选择器,用于选择合适的worker执行任务,可选值: random, round_robin, lower_weight | +| master.max-heartbeat-interval | 10s | master最大心跳间隔 | +| master.task-commit-retry-times | 5 | 任务重试次数 | +| master.task-commit-interval | 1000 | 任务提交间隔,单位为毫秒 | +| master.state-wheel-interval | 5 | 轮询检查状态时间 | +| master.server-load-protection.enabled | true | 是否开启系统保护策略 | +| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | master最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统CPU | +| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | master最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的JVM CPU | +| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | master最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统内存 | +| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | master最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,master服务才能调度任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 | +| master.failover-interval | 10 | failover间隔，单位为分钟 | +| master.kill-application-when-task-failover | true | 当任务实例failover时，是否kill掉yarn或k8s application | +| master.registry-disconnect-strategy.strategy | stop | 当Master与注册中心失联之后采取的策略, 默认值是: stop. 可选值包括： stop, waiting | +| master.registry-disconnect-strategy.max-waiting-time | 100s | 当Master与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Master与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Master会丢弃目前正在执行的工作流，值为0表示会无限期等待 | +| master.master.worker-group-refresh-interval | 10s | 定期将workerGroup从数据库中同步到内存的时间间隔 | +| master.command-fetch-strategy.type | ID_SLOT_BASED | Command拉取策略, 目前仅支持 `ID_SLOT_BASED` | +| master.command-fetch-strategy.config.id-step | 1 | 数据库中t_ds_command的id自增步长 | +| master.command-fetch-strategy.config.fetch-size | 10 | master拉取command数量 | ## Worker Server相关配置 diff --git a/docs/docs/zh/contribute/frontend-development.md b/docs/docs/zh/contribute/frontend-development.md index 42eb2973dd..249b17d546 100644 --- a/docs/docs/zh/contribute/frontend-development.md +++ b/docs/docs/zh/contribute/frontend-development.md @@ -33,7 +33,7 @@ Node包下载 (注意版本 v12.20.2) `https://nodejs.org/download/release/v12.2 > 如果 `npm install` 速度非常慢，你可以设置淘宝镜像 ``` -npm config set registry http://registry.npm.taobao.org/ +npm config set registry http://registry.npmmirror.com/ ``` - 修改 `dolphinscheduler-ui/.env` 文件中的 `API_BASE`，用于跟后端交互： diff --git a/docs/docs/zh/faq.md b/docs/docs/zh/faq.md index 13e0dc8b84..a4f32d08a8 100644 --- a/docs/docs/zh/faq.md +++ b/docs/docs/zh/faq.md @@ -430,11 +430,11 @@ A：1，cd dolphinscheduler-ui 然后删除 node_modules 目录 sudo rm -rf node_modules ``` -​ 2，通过 npm.taobao.org 下载 node-sass +​ 2，通过 npmmirror.com 下载 node-sass ``` sudo npm uninstall node-sass -sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ +sudo npm i node-sass --sass_binary_site=https://npmmirror.com/mirrors/node-sass/ ``` 3，如果步骤 2 报错，请重新构建 node-saas [参考链接](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/zh/contribute/frontend-development.md) diff --git a/docs/docs/zh/guide/installation/kubernetes.md b/docs/docs/zh/guide/installation/kubernetes.md index 20cd8907e8..f4b95de27b 100644 --- a/docs/docs/zh/guide/installation/kubernetes.md +++ b/docs/docs/zh/guide/installation/kubernetes.md @@ -14,16 +14,15 @@ Kubernetes 部署目的是在 Kubernetes 集群中部署 DolphinScheduler 服务 ## 安装 dolphinscheduler -请下载源码包 apache-dolphinscheduler--src.tar.gz，下载地址: [下载](https://dolphinscheduler.apache.org/zh-cn/download) - -发布一个名为 `dolphinscheduler` 的版本(release)，请执行以下命令： - -``` -$ tar -zxvf apache-dolphinscheduler--src.tar.gz -$ cd apache-dolphinscheduler--src/deploy/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . --set image.tag= +```bash +# 自行选择对应的版本 +export VERSION=3.2.1 +helm pull oci://registry-1.docker.io/apache/dolphinscheduler-helm --version ${VERSION} +tar -xvf dolphinscheduler-helm-${VERSION}.tgz +cd dolphinscheduler-helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency update . +helm install dolphinscheduler . ``` 将名为 `dolphinscheduler` 的版本(release) 发布到 `test` 的命名空间中： diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 13479e0d9e..a199167e04 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -118,7 +118,6 @@ export SPRING_DATASOURCE_PASSWORD={password} # DolphinScheduler server related configuration export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} -export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} diff --git a/docs/docs/zh/guide/task/spark.md b/docs/docs/zh/guide/task/spark.md index a392f55826..2f7b2ee346 100644 --- a/docs/docs/zh/guide/task/spark.md +++ b/docs/docs/zh/guide/task/spark.md @@ -23,6 +23,7 @@ Spark 任务类型用于执行 Spark 应用。对于 Spark 节点，worker 支 - 程序类型：支持 Java、Scala、Python 和 SQL 四种语言。 - 主函数的 Class：Spark 程序的入口 Main class 的全路径。 - 主程序包：执行 Spark 程序的 jar 包（通过资源中心上传）。 +- Master：执行 Spark 集群的 Master Url。 - SQL脚本：Spark sql 运行的 .sql 文件中的 SQL 语句。 - 部署方式：(1) spark submit 支持 cluster、client 和 local 三种模式。 (2) spark sql 支持 client 和 local 两种模式。 diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index c0070ac11c..c8ded8cfad 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -48,6 +48,8 @@ import java.util.Objects; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -189,7 +191,7 @@ public final class DingTalkSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -317,15 +319,17 @@ public final class DingTalkSender { String sign = org.apache.commons.lang3.StringUtils.EMPTY; try { Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); - byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); - sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + byte[] signData = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); + sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error("generate sign error, message:{}", e); } return url + "×tamp=" + timestamp + "&sign=" + sign; } + @Getter + @Setter static final class DingTalkSendMsgResponse { private Integer errcode; @@ -334,22 +338,6 @@ public final class DingTalkSender { public DingTalkSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - @Override public boolean equals(final Object o) { if (o == this) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index 7a8df0549c..cd30105c7a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; import org.apache.dolphinscheduler.alert.api.AlertResult; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -47,7 +48,7 @@ public class DingTalkSenderTest { @Test public void testSend() { DingTalkSender dingTalkSender = new DingTalkSender(dingTalkConfig); - dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8"); + dingTalkSender.sendDingTalkMsg("keyWord+Welcome", StandardCharsets.UTF_8.name()); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkSender = new DingTalkSender(dingTalkConfig); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test"); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java index 94eb4efa39..3eec7022fd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java @@ -56,8 +56,6 @@ public final class EmailConstants { public static final String TABLE_BODY_HTML_TAIL = ""; - public static final String UTF_8 = "UTF-8"; - public static final String EXCEL_SUFFIX_XLSX = ".xlsx"; public static final String SINGLE_SLASH = "/"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index 2e400efbce..8826a44fba 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -34,6 +34,7 @@ import org.apache.commons.mail.HtmlEmail; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -171,7 +172,7 @@ public final class MailSender { Session session = getSession(); email.setMailSession(session); email.setFrom(mailSenderEmail); - email.setCharset(EmailConstants.UTF_8); + email.setCharset(StandardCharsets.UTF_8.name()); if (CollectionUtils.isNotEmpty(receivers)) { // receivers mail for (String receiver : receivers) { @@ -344,7 +345,8 @@ public final class MailSender { ExcelUtils.genExcelFile(content, randomFilename, xlsFilePath); part2.attachFile(file); - part2.setFileName(MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, EmailConstants.UTF_8, "B")); + part2.setFileName( + MimeUtility.encodeText(title + EmailConstants.EXCEL_SUFFIX_XLSX, StandardCharsets.UTF_8.name(), "B")); // add components to collection partList.addBodyPart(part1); partList.addBodyPart(part2); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java index f428a16d92..f28df77de9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtilsTest.java @@ -66,8 +66,15 @@ public class ExcelUtilsTest { @Test public void testGenExcelFileByCheckDir() { - ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls"); - File file = new File("/tmp/xls" + EmailConstants.SINGLE_SLASH + "t" + EmailConstants.EXCEL_SUFFIX_XLSX); + String path = "/tmp/xls"; + ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", path); + File file = + new File( + path + + EmailConstants.SINGLE_SLASH + + "t" + + EmailConstants.EXCEL_SUFFIX_XLSX); file.delete(); + Assertions.assertFalse(file.exists()); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index f562999be0..acc255ae0e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -18,10 +18,9 @@ package org.apache.dolphinscheduler.plugin.alert.email; import org.apache.dolphinscheduler.alert.api.AlertConstants; +import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.alert.api.ShowType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate; -import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate; import java.util.ArrayList; import java.util.HashMap; @@ -42,7 +41,6 @@ public class MailUtilsTest { private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class); static MailSender mailSender; private static Map emailConfig = new HashMap<>(); - private static AlertTemplate alertTemplate; @BeforeAll public static void initEmailConfig() { @@ -59,7 +57,6 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, "347801120@qq.com"); emailConfig.put(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, "347801120@qq.com"); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TEXT.getDescp()); - alertTemplate = new DefaultHTMLTemplate(); mailSender = new MailSender(emailConfig); } @@ -77,9 +74,10 @@ public class MailUtilsTest { + "\"Host: 192.168.xx.xx\"," + "\"Notify group :4\"]"; - mailSender.sendMails( + AlertResult alertResult = mailSender.sendMails( "Mysql Exception", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -108,7 +106,8 @@ public class MailUtilsTest { emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "user"); emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "passwd"); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } public String list2String() { @@ -134,7 +133,6 @@ public class MailUtilsTest { logger.info(mapjson); return mapjson; - } @Test @@ -143,7 +141,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails(title, content); + AlertResult alertResult = mailSender.sendMails(title, content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -151,7 +150,8 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } @Test @@ -159,7 +159,7 @@ public class MailUtilsTest { String content = list2String(); emailConfig.put(AlertConstants.NAME_SHOW_TYPE, ShowType.TABLE_ATTACHMENT.getDescp()); mailSender = new MailSender(emailConfig); - mailSender.sendMails("gaojing", content); + AlertResult alertResult = mailSender.sendMails("gaojing", content); + Assertions.assertEquals("false", alertResult.getStatus()); } - } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index 14a1d63ff0..369060843c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -31,6 +31,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -161,7 +162,7 @@ public final class FeiShuSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java index 999a0c9599..a1de852407 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpSender.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -58,7 +59,6 @@ public final class HttpSender { * request type get */ private static final String REQUEST_TYPE_GET = "GET"; - private static final String DEFAULT_CHARSET = "utf-8"; private final String headerParams; private final String bodyParams; private final String contentField; @@ -124,7 +124,7 @@ public final class HttpSender { CloseableHttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, DEFAULT_CHARSET); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } private void createHttpRequest(String msg) throws MalformedURLException, URISyntaxException { @@ -157,7 +157,8 @@ public final class HttpSender { type = URL_SPLICE_CHAR; } try { - url = String.format("%s%s%s=%s", url, type, contentField, URLEncoder.encode(msg, DEFAULT_CHARSET)); + url = String.format("%s%s%s=%s", url, type, contentField, + URLEncoder.encode(msg, StandardCharsets.UTF_8.name())); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -190,7 +191,7 @@ public final class HttpSender { } // set msg content field objectNode.put(contentField, msg); - StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), DEFAULT_CHARSET); + StringEntity entity = new StringEntity(JSONUtils.toJsonString(objectNode), StandardCharsets.UTF_8); ((HttpPost) httpRequest).setEntity(entity); } catch (Exception e) { log.error("send http alert msg exception : {}", e.getMessage()); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java index d27745049e..1106e6799f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/src/main/java/org/apache/dolphinscheduler/plugin/alert/prometheus/PrometheusAlertSender.java @@ -34,6 +34,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -90,7 +91,7 @@ public class PrometheusAlertSender { } HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); log.error( "Prometheus alert manager send alert failed, http status code: {}, title: {} ,content: {}, resp: {}", diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java index a34f062264..3d85d5d638 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtilsTest.java @@ -33,6 +33,7 @@ public class ProcessUtilsTest { @Test public void testExecuteScript() { - ProcessUtils.executeScript(cmd); + int code = ProcessUtils.executeScript(cmd); + assert code != -1; } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java index 60b2f8281a..4096ccc998 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java @@ -29,6 +29,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -81,11 +82,11 @@ public final class SlackSender { } HttpPost httpPost = new HttpPost(webHookUrl); - httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), "UTF-8")); + httpPost.setEntity(new StringEntity(JSONUtils.toJsonString(paramMap), StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, "UTF-8"); + return EntityUtils.toString(entity, StandardCharsets.UTF_8); } catch (Exception e) { log.error("Send message to slack error.", e); return "System Exception"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 8aba9f5c2b..129bc62c1c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -153,7 +153,7 @@ public final class TelegramSender { String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "UTF-8"); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java index 7f5eaef4f9..76ad480015 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java @@ -23,8 +23,6 @@ public final class WeChatAlertConstants { static final String MARKDOWN_ENTER = "\n"; - static final String CHARSET = "UTF-8"; - static final String WE_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}"; static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index 4b49e0436d..c5ffec1f46 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -38,6 +38,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -45,6 +46,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -85,12 +88,12 @@ public final class WeChatSender { CloseableHttpClient httpClient = HttpClients.custom().setRetryHandler(HttpServiceRetryStrategy.retryStrategy).build()) { HttpPost httpPost = new HttpPost(url); - httpPost.setEntity(new StringEntity(data, WeChatAlertConstants.CHARSET)); + httpPost.setEntity(new StringEntity(data, StandardCharsets.UTF_8)); CloseableHttpResponse response = httpClient.execute(httpPost); String resp; try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -142,7 +145,7 @@ public final class WeChatSender { HttpGet httpGet = new HttpGet(url); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, WeChatAlertConstants.CHARSET); + resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); EntityUtils.consume(entity); } @@ -259,6 +262,8 @@ public final class WeChatSender { return null; } + @Getter + @Setter static final class WeChatSendMsgResponse { private Integer errcode; @@ -267,22 +272,6 @@ public final class WeChatSender { public WeChatSendMsgResponse() { } - public Integer getErrcode() { - return this.errcode; - } - - public void setErrcode(Integer errcode) { - this.errcode = errcode; - } - - public String getErrmsg() { - return this.errmsg; - } - - public void setErrmsg(String errmsg) { - this.errmsg = errmsg; - } - public boolean equals(final Object o) { if (o == this) { return true; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 6f39096221..bf28193b3f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -52,6 +52,7 @@ ${basedir}/../../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index fd3d4b02e3..55c5c3446c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -23,25 +23,29 @@ import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient; import org.apache.dolphinscheduler.alert.rpc.AlertRpcServer; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; import org.apache.dolphinscheduler.alert.service.ListenerEventPostService; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.event.EventListener; +import org.springframework.context.annotation.Import; -@SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") @Slf4j +@Import({CommonConfiguration.class, + DaoConfiguration.class, + RegistryConfiguration.class}) +@SpringBootApplication public class AlertServer { @Autowired @@ -59,11 +63,11 @@ public class AlertServer { AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); Thread.currentThread().setName(Constants.THREAD_NAME_ALERT_SERVER); - new SpringApplicationBuilder(AlertServer.class).run(args); + SpringApplication.run(AlertServer.class, args); } - @EventListener - public void run(ApplicationReadyEvent readyEvent) { + @PostConstruct + public void run() { log.info("Alert server is staring ..."); alertPluginManager.start(); alertRegistryClient.start(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 0b28d6c8b0..0dbb6988ce 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -30,15 +30,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 # Mybatis-plus configuration, you don't need to change it mybatis-plus: diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index 77b2f54c64..c9fdab4d51 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index c7e6d9778f..6f7d8f43d2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -18,12 +18,17 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; +import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; +import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; +import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; +import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -37,18 +42,19 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.servlet.ServletComponentScan; -import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; import org.springframework.context.event.EventListener; +@Slf4j +@Import({DaoConfiguration.class, + CommonConfiguration.class, + ServiceConfiguration.class, + StorageConfiguration.class, + RegistryConfiguration.class}) @ServletComponentScan @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler") -@Slf4j public class ApiApplicationServer { - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private PluginDao pluginDao; @@ -62,8 +68,8 @@ public class ApiApplicationServer { public void run(ApplicationReadyEvent readyEvent) { log.info("Received spring application context ready event will load taskPlugin and write to DB"); // install task plugin - taskPluginManager.loadPlugin(); - for (Map.Entry entry : taskPluginManager.getTaskChannelFactoryMap().entrySet()) { + TaskPluginManager.loadPlugin(); + for (Map.Entry entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { String taskPluginName = entry.getKey(); TaskChannelFactory taskChannelFactory = entry.getValue(); List params = taskChannelFactory.getParams(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java deleted file mode 100644 index 0d936f0ad5..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.audit; - -import org.apache.dolphinscheduler.common.enums.AuditOperationType; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; -import org.apache.dolphinscheduler.dao.entity.User; - -import java.util.Date; - -public class AuditMessage { - - private User user; - - private Date auditDate; - - private AuditResourceType resourceType; - - private AuditOperationType operation; - - private Integer resourceId; - - public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation, - Integer resourceId) { - this.user = user; - this.auditDate = auditDate; - this.resourceType = resourceType; - this.operation = operation; - this.resourceId = resourceId; - } - - public User getUser() { - return user; - } - - public void setUser(User user) { - this.user = user; - } - - public Date getAuditDate() { - return auditDate; - } - - public void setAuditDate(Date auditDate) { - this.auditDate = auditDate; - } - - public AuditResourceType getResourceType() { - return resourceType; - } - - public void setResourceType(AuditResourceType resourceType) { - this.resourceType = resourceType; - } - - public AuditOperationType getOperation() { - return operation; - } - - public void setOperation(AuditOperationType operation) { - this.operation = operation; - } - - public Integer getResourceId() { - return resourceId; - } - - public void setResourceId(Integer resourceId) { - this.resourceId = resourceId; - } - - @Override - public String toString() { - return "AuditMessage{" - + "user=" + user - + ", Date=" + auditDate - + ", resourceType" + resourceType - + ", operation=" + operation - + ", resourceId='" + resourceId + '\''; - } -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditPublishService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditPublishService.java deleted file mode 100644 index 34a5cd8ac0..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditPublishService.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.audit; - -import org.apache.dolphinscheduler.api.configuration.ApiConfig; - -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import javax.annotation.PostConstruct; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class AuditPublishService { - - private final BlockingQueue auditMessageQueue = new LinkedBlockingQueue<>(); - - @Autowired - private List subscribers; - - @Autowired - private ApiConfig apiConfig; - - /** - * create a daemon thread to process the message queue - */ - @PostConstruct - private void init() { - if (apiConfig.isAuditEnable()) { - Thread thread = new Thread(this::doPublish); - thread.setDaemon(true); - thread.setName("Audit-Log-Consume-Thread"); - thread.start(); - } - } - - /** - * publish a new audit message - * - * @param message audit message - */ - public void publish(AuditMessage message) { - if (apiConfig.isAuditEnable() && !auditMessageQueue.offer(message)) { - log.error("Publish audit message failed, message:{}", message); - } - } - - /** - * subscribers execute the message processor method - */ - private void doPublish() { - AuditMessage message = null; - while (true) { - try { - message = auditMessageQueue.take(); - for (AuditSubscriber subscriber : subscribers) { - try { - subscriber.execute(message); - } catch (Exception e) { - log.error("Consume audit message failed, message:{}", message, e); - } - } - } catch (InterruptedException e) { - log.error("Consume audit message failed, message:{}", message, e); - Thread.currentThread().interrupt(); - break; - } - } - } - -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLog.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLog.java new file mode 100644 index 0000000000..440a67bf87 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLog.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit; + +import org.apache.dolphinscheduler.api.audit.enums.AuditType; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Custom annotation for logging and auditing operator actions in the system. + * This annotation can be applied to methods to indicate the type of operation, object type, + * and specific parameters to be recorded in the logs. + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface OperatorLog { + + AuditType auditType(); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java new file mode 100644 index 0000000000..aaf35d5b66 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorLogAspect.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit; + +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.AfterReturning; +import org.aspectj.lang.annotation.AfterThrowing; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.annotation.Pointcut; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.stereotype.Component; + +import io.swagger.v3.oas.annotations.Operation; + +@Aspect +@Slf4j +@Component +public class OperatorLogAspect { + + private static final ThreadLocal auditThreadLocal = new ThreadLocal<>(); + + @Pointcut("@annotation(org.apache.dolphinscheduler.api.audit.OperatorLog)") + public void logPointCut() { + } + + @Before("logPointCut()") + public void before(JoinPoint point) { + MethodSignature signature = (MethodSignature) point.getSignature(); + Method method = signature.getMethod(); + OperatorLog operatorLog = method.getAnnotation(OperatorLog.class); + Operation operation = method.getAnnotation(Operation.class); + + if (operation == null) { + log.warn("Operation is null of method: {}", method.getName()); + return; + } + + Map paramsMap = OperatorUtils.getParamsMap(point, signature); + User user = OperatorUtils.getUser(paramsMap); + if (user == null) { + log.error("user is null"); + return; + } + + AuditType auditType = operatorLog.auditType(); + + try { + AuditOperator operator = SpringApplicationContext.getBean(operatorLog.auditType().getOperatorClass()); + List auditLogList = OperatorUtils.buildAuditLogList(operation.description(), auditType, user); + operator.setRequestParam(auditType, auditLogList, paramsMap); + AuditContext auditContext = + new AuditContext(auditLogList, paramsMap, operatorLog, System.currentTimeMillis(), operator); + auditThreadLocal.set(auditContext); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } + } + + @AfterReturning(value = "logPointCut()", returning = "returnValue") + public void afterReturning(Object returnValue) { + try { + AuditContext auditContext = auditThreadLocal.get(); + if (auditContext == null) { + return; + } + auditContext.getOperator().recordAudit(auditContext, returnValue); + } catch (Throwable throwable) { + log.error("Record audit log error", throwable); + } finally { + auditThreadLocal.remove(); + } + } + + @AfterThrowing("logPointCut()") + public void afterThrowing() { + auditThreadLocal.remove(); + } + + @Getter + @Setter + @AllArgsConstructor + public static class AuditContext { + + List auditLogList; + Map paramsMap; + OperatorLog operatorLog; + long beginTime; + AuditOperator operator; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java new file mode 100644 index 0000000000..8dc628b576 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/OperatorUtils.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit; + +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.enums.ExecuteType; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.enums.AuditModelType; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.reflect.MethodSignature; + +@Slf4j +public class OperatorUtils { + + public static boolean resultFail(Result result) { + return result != null && result.isFailed(); + } + + public static List buildAuditLogList(String apiDescription, AuditType auditType, User user) { + List auditLogList = new ArrayList<>(); + AuditLog auditLog = new AuditLog(); + auditLog.setUserId(user.getId()); + auditLog.setModelType(auditType.getAuditModelType().getName()); + auditLog.setOperationType(auditType.getAuditOperationType().getName()); + auditLog.setDescription(apiDescription); + auditLog.setCreateTime(new Date()); + auditLogList.add(auditLog); + return auditLogList; + } + + public static User getUser(Map paramsMap) { + for (Object object : paramsMap.values()) { + if (object instanceof User) { + return (User) object; + } + } + + return null; + } + + public static Map getParamsMap(JoinPoint point, MethodSignature signature) { + Object[] args = point.getArgs(); + String[] strings = signature.getParameterNames(); + + Map paramsMap = new HashMap<>(); + for (int i = 0; i < strings.length; i++) { + paramsMap.put(strings[i], args[i]); + } + + return paramsMap; + } + + public static AuditOperationType modifyReleaseOperationType(AuditType auditType, Map paramsMap) { + switch (auditType.getAuditOperationType()) { + case RELEASE: + ReleaseState releaseState = (ReleaseState) paramsMap.get(Constants.RELEASE_STATE); + if (releaseState == null) { + break; + } + switch (releaseState) { + case ONLINE: + return AuditOperationType.ONLINE; + case OFFLINE: + return AuditOperationType.OFFLINE; + default: + break; + } + break; + case EXECUTE: + ExecuteType executeType = (ExecuteType) paramsMap.get(Constants.EXECUTE_TYPE); + if (executeType == null) { + break; + } + switch (executeType) { + case REPEAT_RUNNING: + return AuditOperationType.RERUN; + case RECOVER_SUSPENDED_PROCESS: + return AuditOperationType.RESUME_PAUSE; + case START_FAILURE_TASK_PROCESS: + return AuditOperationType.RESUME_FAILURE; + case STOP: + return AuditOperationType.STOP; + case PAUSE: + return AuditOperationType.PAUSE; + case EXECUTE_TASK: + return AuditOperationType.EXECUTE; + default: + break; + } + break; + default: + break; + } + + return auditType.getAuditOperationType(); + } + + public static long getObjectIdentityByParma(String[] paramNameArr, Map paramsMap) { + for (String name : paramNameArr) { + if (paramsMap.get(name) instanceof String) { + String param = (String) paramsMap.get(name); + try { + if (param.matches("\\d+")) { + return Long.parseLong(param); + } + } catch (NumberFormatException e) { + return -1; + } + } + } + + return -1; + } + + public static Map getObjectIfFromReturnObject(Object obj, String[] params) { + Map map = new HashMap<>(); + + try { + Class clazz = obj.getClass(); + + if (clazz.equals(Long.class)) { + map.put(params[0], obj); + } + + while (clazz != null) { + Field[] fields = clazz.getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + + if (field.getName().equals(params[0])) { + map.put(params[0], field.get(obj)); + } + } + + clazz = clazz.getSuperclass(); + } + } catch (Exception e) { + log.error("get object if from return object error", e); + } + + return map; + } + + public static boolean isUdfResource(Map paramsMap) { + ResourceType resourceType = (ResourceType) paramsMap.get(Constants.STRING_PLUGIN_PARAM_TYPE); + return resourceType != null && resourceType.equals(ResourceType.UDF); + } + + public static boolean isFolder(String name) { + return name != null && name.endsWith("/"); + } + + public static String getFileAuditObject(AuditType auditType, Map paramsMap, String name) { + boolean isUdfResource = isUdfResource(paramsMap); + boolean isFolder = auditType == AuditType.FOLDER_CREATE || isFolder(name); + if (isUdfResource) { + return isFolder ? AuditModelType.UDF_FOLDER.getName() : AuditModelType.UDF_FILE.getName(); + } else { + return isFolder ? AuditModelType.FOLDER.getName() : AuditModelType.FILE.getName(); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/constants/AuditLogConstants.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/constants/AuditLogConstants.java new file mode 100644 index 0000000000..67df788f1e --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/constants/AuditLogConstants.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.constants; + +public final class AuditLogConstants { + + private AuditLogConstants() { + throw new UnsupportedOperationException("Construct Constants"); + } + + public static final String CODE = "code"; + public static final String CODES = "codes"; + public static final String VERSION = "version"; + public static final String PROCESS_DEFINITION_CODE = "processDefinitionCode"; + public static final String PROCESS_DEFINITION_CODES = "processDefinitionCodes"; + public static final String PROCESS_INSTANCE_IDS = "processInstanceIds"; + public static final String PROCESS_INSTANCE_ID = "processInstanceId"; + public static final String WORKFLOW_DEFINITION_CODE = "workflowDefinitionCode"; + public static final String TYPE = "type"; + public static final String NAME = "name"; + public static final String ID = "id"; + public static final String USER_ID = "userId"; + public static final String QUEUE_ID = "queueId"; + public static final String PRIORITY = "priority"; + public static final String CLUSTER_CODE = "clusterCode"; + public static final String ENVIRONMENT_CODE = "environmentCode"; + public static final String ALIAS = "alias"; + public static final String FILE_NAME = "fileName"; + public static final String FULL_NAME = "fullName"; + public static final String FUNC_NAME = "funcName"; + public static final String UDF_FUNC_ID = "udfFuncId"; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/enums/AuditType.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/enums/AuditType.java new file mode 100644 index 0000000000..cba510180e --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/enums/AuditType.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.enums; + +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ALIAS; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CLUSTER_CODE; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CODE; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.CODES; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ENVIRONMENT_CODE; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FILE_NAME; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FULL_NAME; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.FUNC_NAME; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.ID; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.NAME; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PRIORITY; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_DEFINITION_CODE; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_DEFINITION_CODES; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_INSTANCE_ID; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.PROCESS_INSTANCE_IDS; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.QUEUE_ID; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.TYPE; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.UDF_FUNC_ID; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.USER_ID; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.VERSION; +import static org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants.WORKFLOW_DEFINITION_CODE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.ALARM_GROUP; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.ALARM_INSTANCE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.CLUSTER; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.DATASOURCE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.ENVIRONMENT; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.FILE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.FOLDER; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.K8S_NAMESPACE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROCESS; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROCESS_INSTANCE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.PROJECT; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.SCHEDULE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK_GROUP; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.TASK_INSTANCE; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.TENANT; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.TOKEN; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.UDF_FUNCTION; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.USER; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.WORKER_GROUP; +import static org.apache.dolphinscheduler.common.enums.AuditModelType.YARN_QUEUE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_DELETE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_RERUN; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.BATCH_START; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.CLOSE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.COPY; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.CREATE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.DELETE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.DELETE_VERSION; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.EXECUTE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.EXPORT; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.FORCE_SUCCESS; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.IMPORT; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.MODIFY; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.OFFLINE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.ONLINE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.RELEASE; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.START; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.SWITCH_VERSION; +import static org.apache.dolphinscheduler.common.enums.AuditOperationType.UPDATE; + +import org.apache.dolphinscheduler.api.audit.operator.AuditOperator; +import org.apache.dolphinscheduler.api.audit.operator.impl.AlertGroupAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.AlertInstanceAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ClusterAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.DatasourceAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.EnvironmentAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.K8SNamespaceAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ProcessAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ProcessInstanceAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ProjectAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ResourceAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.ScheduleAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.TaskAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.TaskGroupAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.TaskInstancesAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.TenantAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.TokenAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.UdfFunctionAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.UserAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.WorkerGroupAuditOperatorImpl; +import org.apache.dolphinscheduler.api.audit.operator.impl.YarnQueueAuditOperatorImpl; +import org.apache.dolphinscheduler.common.enums.AuditModelType; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; + +import lombok.Getter; + +@Getter +public enum AuditType { + + PROJECT_CREATE(PROJECT, CREATE, ProjectAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + PROJECT_UPDATE(PROJECT, UPDATE, ProjectAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + PROJECT_DELETE(PROJECT, DELETE, ProjectAuditOperatorImpl.class, new String[]{CODE}, new String[]{}), + + PROCESS_CREATE(PROCESS, CREATE, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + PROCESS_UPDATE(PROCESS, UPDATE, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + PROCESS_SWITCH_VERSION(PROCESS, SWITCH_VERSION, ProcessAuditOperatorImpl.class, new String[]{CODE, VERSION}, + new String[]{}), + PROCESS_DELETE_VERSION(PROCESS, DELETE_VERSION, ProcessAuditOperatorImpl.class, new String[]{CODE, VERSION}, + new String[]{}), + PROCESS_RELEASE(PROCESS, RELEASE, ProcessAuditOperatorImpl.class, new String[]{WORKFLOW_DEFINITION_CODE}, + new String[]{}), + PROCESS_COPY(PROCESS, COPY, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}), + PROCESS_EXPORT(PROCESS, EXPORT, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}), + PROCESS_DELETE(PROCESS, DELETE, ProcessAuditOperatorImpl.class, new String[]{CODE}, new String[]{}), + PROCESS_BATCH_DELETE(PROCESS, BATCH_DELETE, ProcessAuditOperatorImpl.class, new String[]{CODES}, new String[]{}), + PROCESS_START(PROCESS, START, ProcessAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODE}, + new String[]{}), + PROCESS_BATCH_START(PROCESS, BATCH_START, ProcessAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODES}, + new String[]{}), + PROCESS_BATCH_RERUN(PROCESS, BATCH_RERUN, ProcessInstanceAuditOperatorImpl.class, + new String[]{PROCESS_INSTANCE_IDS}, + new String[]{}), + PROCESS_EXECUTE(PROCESS, EXECUTE, ProcessInstanceAuditOperatorImpl.class, new String[]{PROCESS_INSTANCE_ID}, + new String[]{}), + PROCESS_IMPORT(PROCESS, IMPORT, ProcessAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + PROCESS_INSTANCE_UPDATE(PROCESS_INSTANCE, UPDATE, ProcessInstanceAuditOperatorImpl.class, new String[]{ID}, + new String[]{}), + PROCESS_INSTANCE_DELETE(PROCESS_INSTANCE, DELETE, ProcessInstanceAuditOperatorImpl.class, new String[]{ID}, + new String[]{}), + PROCESS_INSTANCE_BATCH_DELETE(PROCESS_INSTANCE, BATCH_DELETE, ProcessInstanceAuditOperatorImpl.class, + new String[]{PROCESS_INSTANCE_IDS}, new String[]{}), + + TASK_CREATE(TASK, CREATE, TaskAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + TASK_UPDATE(TASK, UPDATE, TaskAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + TASK_SWITCH_VERSION(TASK, SWITCH_VERSION, TaskAuditOperatorImpl.class, new String[]{CODE, VERSION}, new String[]{}), + TASK_DELETE_VERSION(TASK, DELETE_VERSION, TaskAuditOperatorImpl.class, new String[]{CODE, VERSION}, new String[]{}), + TASK_DELETE(TASK, DELETE, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}), + TASK_RELEASE(TASK, RELEASE, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}), + TASK_START(TASK, START, TaskAuditOperatorImpl.class, new String[]{CODE}, new String[]{}), + TASK_INSTANCE_FORCE_SUCCESS(TASK_INSTANCE, FORCE_SUCCESS, TaskInstancesAuditOperatorImpl.class, new String[]{ID}, + new String[]{}), + + SCHEDULE_CREATE(SCHEDULE, CREATE, ScheduleAuditOperatorImpl.class, new String[]{PROCESS_DEFINITION_CODE}, + new String[]{ID}), + SCHEDULE_UPDATE(SCHEDULE, UPDATE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + SCHEDULE_ONLINE(SCHEDULE, ONLINE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + SCHEDULE_OFFLINE(SCHEDULE, OFFLINE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + SCHEDULE_DELETE(SCHEDULE, DELETE, ScheduleAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + FOLDER_CREATE(FOLDER, CREATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, ALIAS}, new String[]{}), + FILE_CREATE(FILE, CREATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, FILE_NAME, ALIAS}, new String[]{}), + FILE_UPDATE(FILE, UPDATE, ResourceAuditOperatorImpl.class, new String[]{TYPE, FILE_NAME, ALIAS}, new String[]{}), + FILE_DELETE(FILE, DELETE, ResourceAuditOperatorImpl.class, new String[]{FULL_NAME}, new String[]{}), + + UDF_FUNCTION_CREATE(UDF_FUNCTION, CREATE, UdfFunctionAuditOperatorImpl.class, new String[]{FUNC_NAME}, + new String[]{}), + UDF_FUNCTION_UPDATE(UDF_FUNCTION, UPDATE, UdfFunctionAuditOperatorImpl.class, new String[]{FUNC_NAME}, + new String[]{}), + UDF_FUNCTION_DELETE(UDF_FUNCTION, DELETE, UdfFunctionAuditOperatorImpl.class, new String[]{UDF_FUNC_ID}, + new String[]{}), + + TASK_GROUP_CREATE(TASK_GROUP, CREATE, TaskGroupAuditOperatorImpl.class, new String[]{NAME}, new String[]{}), + TASK_GROUP_UPDATE(TASK_GROUP, UPDATE, TaskGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + TASK_GROUP_CLOSE(TASK_GROUP, CLOSE, TaskGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + TASK_GROUP_START(TASK_GROUP, START, TaskGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + TASK_GROUP_MODIFY(TASK_GROUP, MODIFY, TaskGroupAuditOperatorImpl.class, new String[]{QUEUE_ID, PRIORITY}, + new String[]{}), + + DATASOURCE_CREATE(DATASOURCE, CREATE, DatasourceAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + DATASOURCE_UPDATE(DATASOURCE, UPDATE, DatasourceAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + DATASOURCE_DELETE(DATASOURCE, DELETE, DatasourceAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + TENANT_CREATE(TENANT, CREATE, TenantAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + TENANT_UPDATE(TENANT, UPDATE, TenantAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + TENANT_DELETE(TENANT, DELETE, TenantAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + USER_CREATE(USER, CREATE, UserAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + USER_UPDATE(USER, UPDATE, UserAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + USER_DELETE(USER, DELETE, UserAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + ALARM_GROUP_CREATE(ALARM_GROUP, CREATE, AlertGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + ALARM_GROUP_UPDATE(ALARM_GROUP, UPDATE, AlertGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + ALARM_GROUP_DELETE(ALARM_GROUP, DELETE, AlertGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + ALARM_INSTANCE_CREATE(ALARM_INSTANCE, CREATE, AlertInstanceAuditOperatorImpl.class, new String[]{}, + new String[]{ID}), + ALARM_INSTANCE_UPDATE(ALARM_INSTANCE, UPDATE, AlertInstanceAuditOperatorImpl.class, new String[]{}, + new String[]{ID}), + ALARM_INSTANCE_DELETE(ALARM_INSTANCE, DELETE, AlertInstanceAuditOperatorImpl.class, new String[]{ID}, + new String[]{}), + + WORKER_GROUP_CREATE(WORKER_GROUP, CREATE, WorkerGroupAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + WORKER_GROUP_DELETE(WORKER_GROUP, DELETE, WorkerGroupAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + YARN_QUEUE_CREATE(YARN_QUEUE, CREATE, YarnQueueAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + YARN_QUEUE_UPDATE(YARN_QUEUE, UPDATE, YarnQueueAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + YARN_QUEUE_DELETE(YARN_QUEUE, DELETE, YarnQueueAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + ENVIRONMENT_CREATE(ENVIRONMENT, CREATE, EnvironmentAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + ENVIRONMENT_UPDATE(ENVIRONMENT, UPDATE, EnvironmentAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + ENVIRONMENT_DELETE(ENVIRONMENT, DELETE, EnvironmentAuditOperatorImpl.class, new String[]{ENVIRONMENT_CODE}, + new String[]{}), + + CLUSTER_CREATE(CLUSTER, CREATE, ClusterAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + CLUSTER_UPDATE(CLUSTER, UPDATE, ClusterAuditOperatorImpl.class, new String[]{}, new String[]{CODE}), + CLUSTER_DELETE(CLUSTER, DELETE, ClusterAuditOperatorImpl.class, new String[]{CLUSTER_CODE}, new String[]{}), + + K8S_NAMESPACE_CREATE(K8S_NAMESPACE, CREATE, K8SNamespaceAuditOperatorImpl.class, new String[]{}, new String[]{ID}), + K8S_NAMESPACE_DELETE(K8S_NAMESPACE, DELETE, K8SNamespaceAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + + TOKEN_CREATE(TOKEN, CREATE, TokenAuditOperatorImpl.class, new String[]{}, new String[]{USER_ID}), + TOKEN_UPDATE(TOKEN, UPDATE, TokenAuditOperatorImpl.class, new String[]{}, new String[]{USER_ID}), + TOKEN_DELETE(TOKEN, DELETE, TokenAuditOperatorImpl.class, new String[]{ID}, new String[]{}), + ; + + private final Class operatorClass; + private final AuditModelType auditModelType; + private final AuditOperationType auditOperationType; + + /** + * The names of the fields in the API request to be recorded. + * Represents an array of key-value pairs, e.g., [ID, "status"]. + */ + private final String[] requestParamName; + + /** + * The names of the fields in the returned object to be recorded. + * Represents an array of field names, e.g., [ID, CODE]. + * Specify the field names to record from the returned object. + */ + private final String[] returnObjectFieldName; + + AuditType(AuditModelType auditModelType, AuditOperationType auditOperationType, + Class operatorClass, String[] requestParamName, String[] returnObjectFieldName) { + this.auditModelType = auditModelType; + this.auditOperationType = auditOperationType; + this.operatorClass = operatorClass; + this.requestParamName = requestParamName; + this.returnObjectFieldName = returnObjectFieldName; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java new file mode 100644 index 0000000000..c3a9a845f5 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/AuditOperator.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator; + +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; + +import java.util.List; +import java.util.Map; + +public interface AuditOperator { + + void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue); + + void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java new file mode 100644 index 0000000000..0ab607da65 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/BaseAuditOperator.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator; + +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.OperatorLogAspect; +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.service.AuditService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.AuditLog; + +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.common.base.Strings; + +@Service +@Slf4j +public abstract class BaseAuditOperator implements AuditOperator { + + @Autowired + private AuditService auditService; + + @Override + public void recordAudit(OperatorLogAspect.AuditContext auditContext, Object returnValue) { + Result result = new Result<>(); + + if (returnValue instanceof Result) { + result = (Result) returnValue; + if (OperatorUtils.resultFail(result)) { + log.error("request fail, code {}", result.getCode()); + return; + } + } + + long latency = System.currentTimeMillis() - auditContext.getBeginTime(); + List auditLogList = auditContext.getAuditLogList(); + + Map paramsMap = auditContext.getParamsMap(); + OperatorLog operatorLog = auditContext.getOperatorLog(); + AuditType auditType = operatorLog.auditType(); + + setObjectIdentityFromReturnObject(auditType, result, auditLogList); + modifyAuditOperationType(auditType, paramsMap, auditLogList); + modifyAuditObjectType(auditType, paramsMap, auditLogList); + + auditLogList.forEach(auditLog -> auditLog.setLatency(latency)); + auditLogList.forEach(auditLog -> auditService.addAudit(auditLog)); + } + + @Override + public void setRequestParam(AuditType auditType, List auditLogList, Map paramsMap) { + String[] paramNameArr = auditType.getRequestParamName(); + + if (paramNameArr.length == 0) { + return; + } + + modifyRequestParams(paramNameArr, paramsMap, auditLogList); + setObjectByParma(paramNameArr, paramsMap, auditLogList); + + if (auditLogList.get(0).getModelId() == null) { + auditLogList.get(0).setModelId(OperatorUtils.getObjectIdentityByParma(paramNameArr, paramsMap)); + } + } + + protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + List auditLogList) { + + String name = paramNameArr[0]; + Object value = paramsMap.get(name); + + if (value == null) { + return; + } + + String objName = getObjectNameFromReturnIdentity(value); + + if (Strings.isNullOrEmpty(objName)) { + auditLogList.get(0).setModelName(value.toString()); + return; + } + + try { + long objectId = Long.parseLong(value.toString()); + auditLogList.get(0).setModelId(objectId); + } catch (NumberFormatException e) { + log.error("value is not long, value: {}", value); + } + + auditLogList.get(0).setModelName(objName); + } + + protected void setObjectByParmaArr(String[] paramNameArr, Map paramsMap, + List auditLogList) { + + AuditLog auditLog = auditLogList.get(0); + for (String param : paramNameArr) { + if (!paramsMap.containsKey(param)) { + continue; + } + + String[] identityArr = ((String) paramsMap.get(param)).split(","); + for (String identityString : identityArr) { + long identity = toLong(identityString); + + String value = getObjectNameFromReturnIdentity(identity); + + if (value == null) { + continue; + } + + auditLog.setModelId(identity); + auditLog.setModelName(value); + auditLogList.add(auditLog); + auditLog = AuditLog.copyNewOne(auditLog); + } + } + auditLogList.remove(0); + } + + protected void setObjectIdentityFromReturnObject(AuditType auditType, Result result, + List auditLogList) { + String[] returnObjectFieldNameArr = auditType.getReturnObjectFieldName(); + if (returnObjectFieldNameArr.length == 0) { + return; + } + Map returnObjectMap = + OperatorUtils.getObjectIfFromReturnObject(result.getData(), returnObjectFieldNameArr); + modifyObjectFromReturnObject(returnObjectFieldNameArr, returnObjectMap, auditLogList); + setObjectNameFromReturnIdentity(auditLogList); + } + + protected void setObjectNameFromReturnIdentity(List auditLogList) { + auditLogList + .forEach(auditLog -> auditLog.setModelName(getObjectNameFromReturnIdentity(auditLog.getModelId()))); + } + + protected void modifyObjectFromReturnObject(String[] params, Map returnObjectMap, + List auditLogList) { + if (returnObjectMap.isEmpty() || returnObjectMap.get(params[0]) == null) { + return; + } + + Long objId = toLong(returnObjectMap.get(params[0])); + + if (objId != -1) { + auditLogList.get(0).setModelId(objId); + } + } + + protected Long toLong(Object str) { + if (str == null) { + return -1L; + } + + return NumberUtils.toLong(str.toString(), -1); + } + + protected String getObjectNameFromReturnIdentity(Object identity) { + return identity.toString(); + } + + protected void modifyRequestParams(String[] paramNameArr, Map paramsMap, + List auditLogList) { + + } + + protected void modifyAuditObjectType(AuditType auditType, Map paramsMap, + List auditLogList) { + + } + + protected void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java similarity index 54% rename from dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberImpl.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java index b5ed361c24..d9d55b71f7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertGroupAuditOperatorImpl.java @@ -15,28 +15,29 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.api.audit; +package org.apache.dolphinscheduler.api.audit.operator.impl; -import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.AlertGroup; +import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; -@Component -public class AuditSubscriberImpl implements AuditSubscriber { +@Service +public class AlertGroupAuditOperatorImpl extends BaseAuditOperator { @Autowired - private AuditLogMapper logMapper; + private AlertGroupMapper alertGroupMapper; @Override - public void execute(AuditMessage message) { - AuditLog auditLog = new AuditLog(); - auditLog.setUserId(message.getUser().getId()); - auditLog.setResourceType(message.getResourceType().getCode()); - auditLog.setOperation(message.getOperation().getCode()); - auditLog.setTime(message.getAuditDate()); - auditLog.setResourceId(message.getResourceId()); - logMapper.insert(auditLog); + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + AlertGroup obj = alertGroupMapper.selectById(objId); + return obj == null ? "" : obj.getGroupName(); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java new file mode 100644 index 0000000000..428007a22b --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/AlertInstanceAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; +import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class AlertInstanceAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private AlertPluginInstanceMapper alertPluginInstanceMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + AlertPluginInstance obj = alertPluginInstanceMapper.selectById(objId); + return obj == null ? "" : obj.getInstanceName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java new file mode 100644 index 0000000000..41cf81fb40 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ClusterAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.Cluster; +import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ClusterAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private ClusterMapper clusterMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + Cluster obj = clusterMapper.queryByClusterCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java new file mode 100644 index 0000000000..7a58cce19c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/DatasourceAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class DatasourceAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private DataSourceMapper dataSourceMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + DataSource obj = dataSourceMapper.selectById(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java new file mode 100644 index 0000000000..fc8301b048 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/EnvironmentAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.Environment; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class EnvironmentAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private EnvironmentMapper environmentMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + Environment obj = environmentMapper.queryByEnvironmentCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java new file mode 100644 index 0000000000..e8bf25ef2d --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/K8SNamespaceAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.K8sNamespace; +import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class K8SNamespaceAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private K8sNamespaceMapper k8sNamespaceMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + K8sNamespace obj = k8sNamespaceMapper.selectById(objId); + return obj == null ? "" : obj.getNamespace(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java new file mode 100644 index 0000000000..ba87229729 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessAuditOperatorImpl.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ProcessAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + @Override + public void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap); + auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName())); + } + + @Override + protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + List auditLogList) { + if (paramNameArr[0].equals(AuditLogConstants.CODES) + || paramNameArr[0].equals(AuditLogConstants.PROCESS_DEFINITION_CODES) + || paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) { + super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList); + } else { + super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + } + if (paramsMap.containsKey(AuditLogConstants.VERSION)) { + if (paramsMap.get(AuditLogConstants.VERSION) != null) { + auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString()); + } else { + auditLogList.get(0).setDetail("latest"); + } + } + } + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + ProcessDefinition obj = processDefinitionMapper.queryByCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java new file mode 100644 index 0000000000..f77ac59733 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProcessInstanceAuditOperatorImpl.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; + +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ProcessInstanceAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private ProcessInstanceMapper processInstanceMapper; + + @Override + public void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap); + auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName())); + } + + @Override + protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + List auditLogList) { + if (paramNameArr[0].equals(AuditLogConstants.PROCESS_INSTANCE_IDS)) { + super.setObjectByParmaArr(paramNameArr, paramsMap, auditLogList); + } else { + super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + } + } + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + int objId = NumberUtils.toInt(identity.toString(), -1); + if (objId == -1) { + return ""; + } + + ProcessInstance obj = processInstanceMapper.queryDetailById(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java new file mode 100644 index 0000000000..186d9df4e1 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ProjectAuditOperatorImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class ProjectAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private ProjectMapper projectMapper; + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + Project obj = projectMapper.queryByCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java new file mode 100644 index 0000000000..f7326bca21 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ResourceAuditOperatorImpl.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.AuditLog; + +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Service; + +@Service +public class ResourceAuditOperatorImpl extends BaseAuditOperator { + + @Override + public void modifyAuditObjectType(AuditType auditType, Map paramsMap, List auditLogList) { + auditLogList.forEach(auditLog -> auditLog + .setModelType(OperatorUtils.getFileAuditObject(auditType, paramsMap, auditLog.getModelName()))); + } + + @Override + protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + List auditLogList) { + + Object objName = getFileNameFromParam(paramNameArr, paramsMap); + + if (objName == null) { + return; + } + + auditLogList.get(0).setModelName(objName.toString()); + } + + private String getFileNameFromParam(String[] paramNameArr, Map paramsMap) { + for (String param : paramNameArr) { + if (!param.equals("type") && paramsMap.containsKey(param)) { + return paramsMap.get(param).toString(); + } + } + + return null; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java new file mode 100644 index 0000000000..731eaa25c7 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/ScheduleAuditOperatorImpl.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ScheduleAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private ScheduleMapper scheduleMapper; + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + @Override + public void modifyRequestParams(String[] paramNameArr, Map paramsMap, List auditLogList) { + if (!paramNameArr[0].equals(AuditLogConstants.ID)) { + return; + } + int id = (int) paramsMap.get(paramNameArr[0]); + Schedule schedule = scheduleMapper.selectById(id); + if (schedule != null) { + paramsMap.put(AuditLogConstants.CODE, schedule.getProcessDefinitionCode()); + paramNameArr[0] = AuditLogConstants.CODE; + auditLogList.forEach(auditLog -> auditLog.setDetail(String.valueOf(id))); + } + } + + @Override + protected void setObjectIdentityFromReturnObject(AuditType auditType, Result result, + List auditLogList) { + String[] returnObjectFieldNameArr = auditType.getReturnObjectFieldName(); + if (returnObjectFieldNameArr.length == 0) { + return; + } + + Map returnObjectMap = + OperatorUtils.getObjectIfFromReturnObject(result.getData(), returnObjectFieldNameArr); + auditLogList + .forEach(auditLog -> auditLog.setDetail(returnObjectMap.get(returnObjectFieldNameArr[0]).toString())); + } + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + ProcessDefinition obj = processDefinitionMapper.queryByCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java new file mode 100644 index 0000000000..202a56cce9 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskAuditOperatorImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.OperatorUtils; +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class TaskAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private TaskDefinitionMapper taskDefinitionMapper; + + @Override + public void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + AuditOperationType auditOperationType = OperatorUtils.modifyReleaseOperationType(auditType, paramsMap); + auditLogList.forEach(auditLog -> auditLog.setOperationType(auditOperationType.getName())); + } + + @Override + protected void setObjectByParma(String[] paramNameArr, Map paramsMap, + List auditLogList) { + + super.setObjectByParma(paramNameArr, paramsMap, auditLogList); + if (paramsMap.containsKey(AuditLogConstants.VERSION)) { + auditLogList.get(0).setDetail(paramsMap.get(AuditLogConstants.VERSION).toString()); + } + } + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + TaskDefinition obj = taskDefinitionMapper.queryByCode(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java new file mode 100644 index 0000000000..f50fef2e8d --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskGroupAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.TaskGroup; +import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class TaskGroupAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private TaskGroupMapper taskGroupMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + TaskGroup obj = taskGroupMapper.selectById(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java new file mode 100644 index 0000000000..1b8067fb0b --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TaskInstancesAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class TaskInstancesAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private TaskInstanceMapper taskInstanceMapper; + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + TaskInstance obj = taskInstanceMapper.selectById(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java new file mode 100644 index 0000000000..4dd80bc422 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TenantAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class TenantAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private TenantMapper tenantMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + Tenant obj = tenantMapper.selectById(objId); + return obj == null ? "" : obj.getTenantCode(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java new file mode 100644 index 0000000000..39c5a4f2f7 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/TokenAuditOperatorImpl.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.AccessToken; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class TokenAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private AccessTokenMapper accessTokenMapper; + + @Autowired + private UserMapper userMapper; + + @Override + public void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + if (paramsMap.get(AuditLogConstants.USER_ID) != null) { + User user = userMapper.selectById(paramsMap.get(AuditLogConstants.USER_ID).toString()); + auditLogList.forEach(auditLog -> { + auditLog.setModelName(user.getUserName()); + auditLog.setModelId(Long.valueOf(user.getId())); + }); + } + } + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + AccessToken obj = accessTokenMapper.selectById(objId); + return obj == null ? "" : obj.getUserName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java new file mode 100644 index 0000000000..9a185621df --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UdfFunctionAuditOperatorImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; + +import org.apache.commons.lang3.math.NumberUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class UdfFunctionAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private UdfFuncMapper udfFuncMapper; + + @Override + protected String getObjectNameFromReturnIdentity(Object identity) { + int objId = NumberUtils.toInt(identity.toString(), -1); + if (objId == -1) { + return ""; + } + + UdfFunc obj = udfFuncMapper.selectUdfById(objId); + return obj == null ? "" : obj.getFuncName(); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java new file mode 100644 index 0000000000..88db6923cd --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/UserAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class UserAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private UserMapper userMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + User obj = userMapper.selectById(objId); + return obj == null ? "" : obj.getUserName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java new file mode 100644 index 0000000000..e576a64474 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/WorkerGroupAuditOperatorImpl.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.constants.AuditLogConstants; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; +import org.apache.dolphinscheduler.dao.entity.AuditLog; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; + +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class WorkerGroupAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private WorkerGroupMapper workerGroupMapper; + + @Override + public void modifyAuditOperationType(AuditType auditType, Map paramsMap, + List auditLogList) { + if (auditType.getAuditOperationType() == AuditOperationType.CREATE + && paramsMap.get(AuditLogConstants.ID) != null && + !paramsMap.get(AuditLogConstants.ID).toString().equals("0")) { + auditLogList.forEach(auditLog -> auditLog.setOperationType(AuditOperationType.UPDATE.getName())); + } + } + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + WorkerGroup obj = workerGroupMapper.selectById(objId); + return obj == null ? "" : obj.getName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java new file mode 100644 index 0000000000..cf7608bb61 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/operator/impl/YarnQueueAuditOperatorImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.audit.operator.impl; + +import org.apache.dolphinscheduler.api.audit.operator.BaseAuditOperator; +import org.apache.dolphinscheduler.dao.entity.Queue; +import org.apache.dolphinscheduler.dao.mapper.QueueMapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class YarnQueueAuditOperatorImpl extends BaseAuditOperator { + + @Autowired + private QueueMapper queueMapper; + + @Override + public String getObjectNameFromReturnIdentity(Object identity) { + Long objId = toLong(identity); + if (objId == -1) { + return ""; + } + + Queue obj = queueMapper.selectById(objId); + return obj == null ? "" : obj.getQueueName(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 781c08ad62..3e594dadbc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_BY_ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_LIST_PAGING_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ACCESS_TOKEN_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AccessTokenService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -83,6 +85,7 @@ public class AccessTokenController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_ACCESS_TOKEN_ERROR) + @OperatorLog(auditType = AuditType.TOKEN_CREATE) public Result createToken(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "userId") int userId, @RequestParam(value = "expireTime") String expireTime, @@ -169,13 +172,15 @@ public class AccessTokenController extends BaseController { * @return delete result code */ @Parameter(hidden = true) + @Operation(summary = "deleteToken", description = "DELETE_TOKEN_NOTES") @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_ACCESS_TOKEN_ERROR) + @OperatorLog(auditType = AuditType.TOKEN_DELETE) public Result delAccessTokenById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id) { accessTokenService.deleteAccessTokenById(loginUser, id); - return Result.success(true); + return Result.success(false); } /** @@ -198,6 +203,7 @@ public class AccessTokenController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_ACCESS_TOKEN_ERROR) + @OperatorLog(auditType = AuditType.TOKEN_UPDATE) public Result updateToken(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id, @RequestParam(value = "userId") int userId, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index ac3bd9376d..3cbaf0a3d4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALERT_GROUP_ERR import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALL_ALERTGROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ALERT_GROUP_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AlertGroupService; @@ -86,6 +88,7 @@ public class AlertGroupController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_ALERT_GROUP_ERROR) + @OperatorLog(auditType = AuditType.ALARM_GROUP_CREATE) public Result createAlertGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "groupName") String groupName, @RequestParam(value = "description", required = false) String description, @@ -195,6 +198,7 @@ public class AlertGroupController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_ALERT_GROUP_ERROR) + @OperatorLog(auditType = AuditType.ALARM_GROUP_UPDATE) public Result updateAlertGroupById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id, @RequestParam(value = "groupName") String groupName, @@ -219,6 +223,7 @@ public class AlertGroupController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_ALERT_GROUP_ERROR) + @OperatorLog(auditType = AuditType.ALARM_GROUP_DELETE) public Result deleteAlertGroupById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id) { alertGroupService.deleteAlertGroupById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java index 143116fc8e..68a46e5537 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java @@ -25,6 +25,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALL_ALERT_PLUGI import static org.apache.dolphinscheduler.api.enums.Status.SEND_TEST_ALERT_PLUGIN_INSTANCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ALERT_PLUGIN_INSTANCE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService; @@ -91,6 +93,7 @@ public class AlertPluginInstanceController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_ALERT_PLUGIN_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.ALARM_INSTANCE_CREATE) public Result createAlertPluginInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "pluginDefineId") int pluginDefineId, @RequestParam(value = "instanceName") String instanceName, @@ -134,6 +137,7 @@ public class AlertPluginInstanceController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_ALERT_PLUGIN_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.ALARM_INSTANCE_UPDATE) public Result updateAlertPluginInstanceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id, @RequestParam(value = "instanceName") String instanceName, @@ -158,6 +162,7 @@ public class AlertPluginInstanceController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_ALERT_PLUGIN_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.ALARM_INSTANCE_DELETE) public Result deleteAlertPluginInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AuditLogController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AuditLogController.java index 6d334951e8..afa418ebb6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AuditLogController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AuditLogController.java @@ -20,15 +20,17 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_AUDIT_LOG_LIST_PAGING; import org.apache.dolphinscheduler.api.dto.AuditDto; +import org.apache.dolphinscheduler.api.dto.auditLog.AuditModelTypeDto; +import org.apache.dolphinscheduler.api.dto.auditLog.AuditOperationTypeDto; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AuditService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AuditOperationType; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; import org.apache.dolphinscheduler.dao.entity.User; +import java.util.List; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; @@ -57,21 +59,23 @@ public class AuditLogController extends BaseController { * * @param loginUser login user * @param pageNo page number - * @param resourceType resource type - * @param operationType operation type + * @param pageSize page size + * @param modelTypes model types + * @param operationTypes operation types + * @param userName user name + * @param modelName model name * @param startDate start time * @param endDate end time - * @param userName user name - * @param pageSize page size * @return audit log content */ @Operation(summary = "queryAuditLogListPaging", description = "QUERY_AUDIT_LOG") @Parameters({ @Parameter(name = "startDate", description = "START_DATE", schema = @Schema(implementation = String.class)), @Parameter(name = "endDate", description = "END_DATE", schema = @Schema(implementation = String.class)), - @Parameter(name = "resourceType", description = "RESOURCE_TYPE", schema = @Schema(implementation = AuditResourceType.class)), - @Parameter(name = "operationType", description = "OPERATION_TYPE", schema = @Schema(implementation = AuditOperationType.class)), + @Parameter(name = "objectTypes", description = "MODEL_TYPES", schema = @Schema(implementation = String.class)), + @Parameter(name = "operationTypes", description = "OPERATION_TYPES", schema = @Schema(implementation = String.class)), @Parameter(name = "userName", description = "USER_NAME", schema = @Schema(implementation = String.class)), + @Parameter(name = "objectName", description = "MODEL_NAME", schema = @Schema(implementation = String.class)), @Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")), @Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20")) }) @@ -81,21 +85,48 @@ public class AuditLogController extends BaseController { public Result> queryAuditLogListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize, - @RequestParam(value = "resourceType", required = false) AuditResourceType resourceType, - @RequestParam(value = "operationType", required = false) AuditOperationType operationType, + @RequestParam(value = "modelTypes", required = false) String modelTypes, + @RequestParam(value = "operationTypes", required = false) String operationTypes, @RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, - @RequestParam(value = "userName", required = false) String userName) { + @RequestParam(value = "userName", required = false) String userName, + @RequestParam(value = "modelName", required = false) String modelName) { checkPageParams(pageNo, pageSize); PageInfo auditDtoPageInfo = auditService.queryLogListPaging( - loginUser, - resourceType, - operationType, + modelTypes, + operationTypes, startDate, endDate, userName, + modelName, pageNo, pageSize); return Result.success(auditDtoPageInfo); } + + /** + * query audit log operation type list + * + * @return object type list + */ + @Operation(summary = "queryAuditOperationTypeList", description = "QUERY_AUDIT_OPERATION_TYPE_LIST") + @GetMapping(value = "/audit-log-operation-type") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_AUDIT_LOG_LIST_PAGING) + public Result> queryAuditOperationTypeList() { + return Result.success(AuditOperationTypeDto.getOperationTypeDtoList()); + } + + /** + * query audit log model type list + * + * @return model type list + */ + @Operation(summary = "queryAuditModelTypeList", description = "QUERY_AUDIT_MODEL_TYPE_LIST") + @GetMapping(value = "/audit-log-model-type") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_AUDIT_LOG_LIST_PAGING) + public Result> queryAuditModelTypeList() { + return Result.success(AuditModelTypeDto.getModelTypeDtoList()); + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java index b29ed9b7a3..3cae7afbda 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_CLUSTER_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_CLUSTER_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_CLUSTER_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.dto.ClusterDto; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ClusterService; @@ -81,6 +83,7 @@ public class ClusterController extends BaseController { @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_CLUSTER_ERROR) + @OperatorLog(auditType = AuditType.CLUSTER_CREATE) public Result createCluster(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("name") String name, @RequestParam("config") String config, @@ -110,6 +113,7 @@ public class ClusterController extends BaseController { @PostMapping(value = "/update") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_CLUSTER_ERROR) + @OperatorLog(auditType = AuditType.CLUSTER_UPDATE) public Result updateCluster(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("code") Long code, @RequestParam("name") String name, @@ -181,6 +185,7 @@ public class ClusterController extends BaseController { @PostMapping(value = "/delete") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_CLUSTER_ERROR) + @OperatorLog(auditType = AuditType.CLUSTER_DELETE) public Result deleteCluster(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("clusterCode") Long clusterCode) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index d4fa83dd3a..90dfaf7c68 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -31,6 +31,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_DATASOUR import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_DATASOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAME_FAILURE; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataSourceService; @@ -92,6 +94,7 @@ public class DataSourceController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_DATASOURCE_ERROR) + @OperatorLog(auditType = AuditType.DATASOURCE_CREATE) public Result createDataSource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "dataSourceParam", description = "DATA_SOURCE_PARAM", required = true) @RequestBody String jsonStr) { BaseDataSourceParamDTO dataSourceParam = DataSourceUtils.buildDatasourceParam(jsonStr); @@ -116,6 +119,7 @@ public class DataSourceController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_DATASOURCE_ERROR) + @OperatorLog(auditType = AuditType.DATASOURCE_UPDATE) public Result updateDataSource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") Integer id, @RequestBody String jsonStr) { @@ -250,6 +254,7 @@ public class DataSourceController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_DATA_SOURCE_FAILURE) + @OperatorLog(auditType = AuditType.DATASOURCE_DELETE) public Result deleteDataSource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable("id") int id) { dataSourceService.delete(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java index 3e33da8909..0520f5d075 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ENVIRONMENT_ERR import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ENVIRONMENT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_ENVIRONMENT_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.EnvironmentService; import org.apache.dolphinscheduler.api.utils.Result; @@ -80,6 +82,7 @@ public class EnvironmentController extends BaseController { @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_ENVIRONMENT_ERROR) + @OperatorLog(auditType = AuditType.ENVIRONMENT_CREATE) public Result createEnvironment(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("name") String name, @RequestParam("config") String config, @@ -111,6 +114,7 @@ public class EnvironmentController extends BaseController { @PostMapping(value = "/update") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_ENVIRONMENT_ERROR) + @OperatorLog(auditType = AuditType.ENVIRONMENT_UPDATE) public Result updateEnvironment(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("code") Long code, @RequestParam("name") String name, @@ -183,6 +187,7 @@ public class EnvironmentController extends BaseController { @PostMapping(value = "/delete") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_ENVIRONMENT_ERROR) + @OperatorLog(auditType = AuditType.ENVIRONMENT_DELETE) public Result deleteEnvironment(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("environmentCode") Long environmentCode) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 98070b4d65..a2a300b946 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -25,6 +25,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_EXECUTING_WORKF import static org.apache.dolphinscheduler.api.enums.Status.START_PROCESS_INSTANCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.START_TASK_INSTANCE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -132,6 +134,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "start-process-instance") @ResponseStatus(HttpStatus.OK) @ApiException(START_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_START) public Result startProcessInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "processDefinitionCode") long processDefinitionCode, @@ -229,6 +232,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "batch-start-process-instance") @ResponseStatus(HttpStatus.OK) @ApiException(BATCH_START_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_BATCH_START) public Result batchStartProcessInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "processDefinitionCodes") String processDefinitionCodes, @@ -318,6 +322,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "/execute") @ResponseStatus(HttpStatus.OK) @ApiException(EXECUTE_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_EXECUTE) public Result execute(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("processInstanceId") Integer processInstanceId, @@ -344,6 +349,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "/batch-execute") @ResponseStatus(HttpStatus.OK) @ApiException(BATCH_EXECUTE_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_BATCH_RERUN) public Result batchExecute(@RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable long projectCode, @RequestParam("processInstanceIds") String processInstanceIds, @@ -440,6 +446,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "/task-instance/{code}/start") @ResponseStatus(HttpStatus.OK) @ApiException(START_TASK_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.TASK_START) public Result startStreamTaskInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @Parameter(name = "code", description = "TASK_CODE", required = true) @PathVariable long code, @@ -482,6 +489,7 @@ public class ExecutorController extends BaseController { @PostMapping(value = "/execute-task") @ResponseStatus(HttpStatus.OK) @ApiException(EXECUTE_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_EXECUTE) public Result executeTask(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("processInstanceId") Integer processInstanceId, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java index 38b1923c82..dbc48a9601 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java @@ -25,6 +25,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_K8S_NAMESPACE_L import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UNAUTHORIZED_NAMESPACE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_K8S_NAMESPACE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.K8sNamespaceService; import org.apache.dolphinscheduler.api.utils.Result; @@ -107,6 +109,7 @@ public class K8sNamespaceController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_K8S_NAMESPACE_ERROR) + @OperatorLog(auditType = AuditType.K8S_NAMESPACE_CREATE) public Result createNamespace(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "namespace") String namespace, @RequestParam(value = "clusterCode") Long clusterCode) { @@ -152,6 +155,7 @@ public class K8sNamespaceController extends BaseController { @PostMapping(value = "/delete") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_K8S_NAMESPACE_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.K8S_NAMESPACE_DELETE) public Result delNamespaceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id") int id) { Map result = k8sNamespaceService.deleteNamespaceById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 261ea9183b..d7ed288ef6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -36,6 +36,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.SWITCH_PROCESS_DEFINI import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROCESS_DEFINITION_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; @@ -112,6 +114,7 @@ public class ProcessDefinitionController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_PROCESS_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_CREATE) public Result createProcessDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "name", required = true) String name, @@ -146,6 +149,7 @@ public class ProcessDefinitionController extends BaseController { @PostMapping(value = "/batch-copy") @ResponseStatus(HttpStatus.OK) @ApiException(BATCH_COPY_PROCESS_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_COPY) public Result copyProcessDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "codes", required = true) String codes, @@ -232,6 +236,7 @@ public class ProcessDefinitionController extends BaseController { @PutMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_PROCESS_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_UPDATE) public Result updateProcessDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "name", required = true) String name, @@ -307,6 +312,7 @@ public class ProcessDefinitionController extends BaseController { @GetMapping(value = "/{code}/versions/{version}") @ResponseStatus(HttpStatus.OK) @ApiException(SWITCH_PROCESS_DEFINITION_VERSION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_SWITCH_VERSION) public Result switchProcessDefinitionVersion(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code, @@ -333,12 +339,12 @@ public class ProcessDefinitionController extends BaseController { @DeleteMapping(value = "/{code}/versions/{version}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_PROCESS_DEFINITION_VERSION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_DELETE_VERSION) public Result deleteProcessDefinitionVersion(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, - @PathVariable(value = "code") long workflowDefinitionCode, - @PathVariable(value = "version") int workflowDefinitionVersion) { - processDefinitionService.deleteProcessDefinitionVersion(loginUser, projectCode, workflowDefinitionCode, - workflowDefinitionVersion); + @PathVariable(value = "code") long code, + @PathVariable(value = "version") int version) { + processDefinitionService.deleteProcessDefinitionVersion(loginUser, projectCode, code, version); return Result.success(); } @@ -351,6 +357,7 @@ public class ProcessDefinitionController extends BaseController { @PostMapping(value = "/{code}/release") @ResponseStatus(HttpStatus.OK) @ApiException(RELEASE_PROCESS_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_RELEASE) public Result releaseProcessDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code", required = true) long workflowDefinitionCode, @@ -611,10 +618,11 @@ public class ProcessDefinitionController extends BaseController { @DeleteMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_PROCESS_DEFINE_BY_CODE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_DELETE) public Result deleteProcessDefinitionByCode(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, - @PathVariable("code") long workflowDefinitionCode) { - processDefinitionService.deleteProcessDefinitionByCode(loginUser, workflowDefinitionCode); + @PathVariable("code") long code) { + processDefinitionService.deleteProcessDefinitionByCode(loginUser, code); return new Result(Status.SUCCESS); } @@ -633,6 +641,7 @@ public class ProcessDefinitionController extends BaseController { @PostMapping(value = "/batch-delete") @ResponseStatus(HttpStatus.OK) @ApiException(BATCH_DELETE_PROCESS_DEFINE_BY_CODES_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_BATCH_DELETE) public Result batchDeleteProcessDefinitionByCodes(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("codes") String codes) { @@ -656,6 +665,7 @@ public class ProcessDefinitionController extends BaseController { }) @PostMapping(value = "/batch-export") @ResponseBody + @OperatorLog(auditType = AuditType.PROCESS_EXPORT) public void batchExportProcessDefinitionByCodes(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("codes") String codes, @@ -699,6 +709,7 @@ public class ProcessDefinitionController extends BaseController { }) @PostMapping(value = "/import") @ApiException(IMPORT_PROCESS_DEFINE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_IMPORT) public Result importProcessDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("file") MultipartFile file) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 85f9cbcf08..6d7b70918f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -173,6 +175,7 @@ public class ProcessInstanceController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(Status.UPDATE_PROCESS_INSTANCE_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_INSTANCE_UPDATE) public Result updateProcessInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "taskRelationJson", required = true) String taskRelationJson, @@ -255,6 +258,7 @@ public class ProcessInstanceController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_INSTANCE_DELETE) public Result deleteProcessInstanceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("id") Integer id) { @@ -388,6 +392,7 @@ public class ProcessInstanceController extends BaseController { @PostMapping(value = "/batch-delete") @ResponseStatus(HttpStatus.OK) @ApiException(Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR) + @OperatorLog(auditType = AuditType.PROCESS_INSTANCE_BATCH_DELETE) public Result batchDeleteProcessInstanceByIds(@RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable long projectCode, @RequestParam("processInstanceIds") String processInstanceIds) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index 7963bae22d..ecec69622a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -27,7 +27,10 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROJECT_DETAILS import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UNAUTHORIZED_PROJECT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROJECT_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.AuditService; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; @@ -67,6 +70,9 @@ public class ProjectController extends BaseController { @Autowired private ProjectService projectService; + @Autowired + private AuditService auditService; + /** * create project * @@ -83,6 +89,7 @@ public class ProjectController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_PROJECT_ERROR) + @OperatorLog(auditType = AuditType.PROJECT_CREATE) public Result createProject(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("projectName") String projectName, @RequestParam(value = "description", required = false) String description) { @@ -107,6 +114,7 @@ public class ProjectController extends BaseController { @PutMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_PROJECT_ERROR) + @OperatorLog(auditType = AuditType.PROJECT_UPDATE) public Result updateProject(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable("code") Long code, @RequestParam("projectName") String projectName, @@ -207,6 +215,7 @@ public class ProjectController extends BaseController { @DeleteMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_PROJECT_ERROR) + @OperatorLog(auditType = AuditType.PROJECT_DELETE) public Result deleteProject(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable("code") Long code) { return projectService.deleteProject(loginUser, code); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java index dbbc7d858e..4e69dede6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java @@ -23,6 +23,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_QUEUE_LIST_ERRO import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_QUEUE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_QUEUE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -124,6 +126,7 @@ public class QueueController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_QUEUE_ERROR) + @OperatorLog(auditType = AuditType.YARN_QUEUE_CREATE) public Result createQueue(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "queue") String queue, @RequestParam(value = "queueName") String queueName) { @@ -148,6 +151,7 @@ public class QueueController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.CREATED) @ApiException(UPDATE_QUEUE_ERROR) + @OperatorLog(auditType = AuditType.YARN_QUEUE_UPDATE) public Result updateQueue(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id, @RequestParam(value = "queue") String queue, @@ -169,6 +173,7 @@ public class QueueController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_QUEUE_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.YARN_QUEUE_DELETE) public Result deleteQueueById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id) throws Exception { queueService.deleteQueueById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index e9c28a5f6e..0338453197 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -37,6 +37,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_N import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ResourcesService; @@ -111,6 +113,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))}) @PostMapping(value = "/directory") @ApiException(CREATE_RESOURCE_ERROR) + @OperatorLog(auditType = AuditType.FOLDER_CREATE) public Result createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "name") String alias, @@ -133,6 +136,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))}) @PostMapping() @ApiException(CREATE_RESOURCE_ERROR) + @OperatorLog(auditType = AuditType.FILE_CREATE) public Result createResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "name") String alias, @@ -160,6 +164,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))}) @PutMapping() @ApiException(UPDATE_RESOURCE_ERROR) + @OperatorLog(auditType = AuditType.FILE_UPDATE) public Result updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "fullName") String fullName, @RequestParam(value = "tenantCode", required = false) String tenantCode, @@ -236,6 +241,7 @@ public class ResourcesController extends BaseController { @DeleteMapping() @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_RESOURCE_ERROR) + @OperatorLog(auditType = AuditType.FILE_DELETE) public Result deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "fullName") String fullName, @RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception { @@ -358,6 +364,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class))}) @PostMapping(value = "/online-create") @ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR) + @OperatorLog(auditType = AuditType.FILE_CREATE) public Result createResourceFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "fileName") String fileName, @@ -385,6 +392,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class))}) @PutMapping(value = "/update-content") @ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR) + @OperatorLog(auditType = AuditType.FILE_UPDATE) public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "fullName") String fullName, @RequestParam(value = "tenantCode") String tenantCode, @@ -445,6 +453,7 @@ public class ResourcesController extends BaseController { @PostMapping(value = "/udf-func") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_UDF_FUNCTION_ERROR) + @OperatorLog(auditType = AuditType.UDF_FUNCTION_CREATE) public Result createUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") UdfType type, @RequestParam(value = "funcName") String funcName, @@ -502,6 +511,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class))}) @PutMapping(value = "/udf-func/{id}") @ApiException(UPDATE_UDF_FUNCTION_ERROR) + @OperatorLog(auditType = AuditType.UDF_FUNCTION_UPDATE) public Result updateUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int udfFuncId, @RequestParam(value = "type") UdfType type, @RequestParam(value = "funcName") String funcName, @@ -590,6 +600,7 @@ public class ResourcesController extends BaseController { @DeleteMapping(value = "/udf-func/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_UDF_FUNCTION_ERROR) + @OperatorLog(auditType = AuditType.UDF_FUNCTION_DELETE) public Result deleteUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int udfFuncId) { return udfFuncService.delete(loginUser, udfFuncId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index 3f3660e377..7dccc67420 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -27,6 +27,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_SCHEDULE_LIST_P import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_SCHEDULE_ERROR; import static org.apache.dolphinscheduler.common.constants.Constants.SESSION_USER; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.SchedulerService; @@ -104,6 +106,7 @@ public class SchedulerController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_SCHEDULE_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_CREATE) public Result createSchedule(@Parameter(hidden = true) @RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "processDefinitionCode") long processDefinitionCode, @@ -161,6 +164,7 @@ public class SchedulerController extends BaseController { @PutMapping("/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_SCHEDULE_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_UPDATE) public Result updateSchedule(@Parameter(hidden = true) @RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "id") Integer id, @@ -185,6 +189,7 @@ public class SchedulerController extends BaseController { }) @PostMapping("/{id}/online") @ApiException(PUBLISH_SCHEDULE_ONLINE_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_ONLINE) public Result publishScheduleOnline(@Parameter(hidden = true) @RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("id") Integer id) { @@ -198,6 +203,7 @@ public class SchedulerController extends BaseController { }) @PostMapping("/{id}/offline") @ApiException(OFFLINE_SCHEDULE_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_OFFLINE) public Result offlineSchedule(@Parameter(hidden = true) @RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("id") Integer id) { @@ -253,6 +259,7 @@ public class SchedulerController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_SCHEDULE_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_DELETE) public Result deleteScheduleById(@RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable("id") Integer id) { @@ -325,6 +332,7 @@ public class SchedulerController extends BaseController { @PutMapping("/update/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_SCHEDULE_ERROR) + @OperatorLog(auditType = AuditType.SCHEDULE_UPDATE) public Result updateScheduleByProcessDefinitionCode(@Parameter(hidden = true) @RequestAttribute(value = SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long processDefinitionCode, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java index 458f747255..fc819c284d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java @@ -28,6 +28,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.RELEASE_TASK_DEFINITI import static org.apache.dolphinscheduler.api.enums.Status.SWITCH_TASK_DEFINITION_VERSION_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TASK_DEFINITION_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TaskDefinitionService; @@ -88,6 +90,7 @@ public class TaskDefinitionController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_TASK_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.TASK_CREATE) public Result createTaskDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "taskDefinitionJson", required = true) String taskDefinitionJson) { @@ -116,6 +119,7 @@ public class TaskDefinitionController extends BaseController { @PostMapping("/save-single") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_TASK_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.TASK_CREATE) public Result createTaskBindsWorkFlow(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "processDefinitionCode", required = true) long processDefinitionCode, @@ -144,6 +148,7 @@ public class TaskDefinitionController extends BaseController { @PutMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_TASK_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.TASK_UPDATE) public Result updateTaskDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code, @@ -173,6 +178,7 @@ public class TaskDefinitionController extends BaseController { @PutMapping(value = "/{code}/with-upstream") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_TASK_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.TASK_UPDATE) public Result updateTaskWithUpstream(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code, @@ -229,6 +235,7 @@ public class TaskDefinitionController extends BaseController { @GetMapping(value = "/{code}/versions/{version}") @ResponseStatus(HttpStatus.OK) @ApiException(SWITCH_TASK_DEFINITION_VERSION_ERROR) + @OperatorLog(auditType = AuditType.TASK_SWITCH_VERSION) public Result switchTaskDefinitionVersion(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code, @@ -254,6 +261,7 @@ public class TaskDefinitionController extends BaseController { @DeleteMapping(value = "/{code}/versions/{version}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_TASK_DEFINITION_VERSION_ERROR) + @OperatorLog(auditType = AuditType.TASK_DELETE_VERSION) public Result deleteTaskDefinitionVersion(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code, @@ -278,6 +286,7 @@ public class TaskDefinitionController extends BaseController { @DeleteMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_TASK_DEFINE_BY_CODE_ERROR) + @OperatorLog(auditType = AuditType.TASK_DELETE) public Result deleteTaskDefinitionByCode(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code") long code) { @@ -384,6 +393,7 @@ public class TaskDefinitionController extends BaseController { @PostMapping(value = "/{code}/release") @ResponseStatus(HttpStatus.OK) @ApiException(RELEASE_TASK_DEFINITION_ERROR) + @OperatorLog(auditType = AuditType.TASK_RELEASE) public Result releaseTaskDefinition(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "code", required = true) long code, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java index 5d8735fbaa..2f99f098ed 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_GROUP_QUEU import static org.apache.dolphinscheduler.api.enums.Status.START_TASK_GROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TASK_GROUP_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TaskGroupQueueService; import org.apache.dolphinscheduler.api.service.TaskGroupService; @@ -81,6 +83,7 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_CREATE) public Result createTaskGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("name") String name, @RequestParam(value = "projectCode", required = false, defaultValue = "0") Long projectCode, @@ -112,6 +115,7 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/update") @ResponseStatus(HttpStatus.CREATED) @ApiException(UPDATE_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_UPDATE) public Result updateTaskGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("id") Integer id, @RequestParam("name") String name, @@ -214,6 +218,7 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/close-task-group") @ResponseStatus(HttpStatus.CREATED) @ApiException(CLOSE_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_CLOSE) public Result closeTaskGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id", required = false) Integer id) { @@ -235,6 +240,7 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/start-task-group") @ResponseStatus(HttpStatus.CREATED) @ApiException(START_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_START) public Result startTaskGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id", required = false) Integer id) { Map result = taskGroupService.startTaskGroup(loginUser, id); @@ -255,9 +261,10 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/forceStart") @ResponseStatus(HttpStatus.CREATED) @ApiException(START_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_START) public Result forceStart(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "queueId") Integer queueId) { - Map result = taskGroupService.forceStartTask(loginUser, queueId); + @RequestParam(value = "queueId") Integer id) { + Map result = taskGroupService.forceStartTask(loginUser, id); return returnDataList(result); } @@ -276,6 +283,7 @@ public class TaskGroupController extends BaseController { @PostMapping(value = "/modifyPriority") @ResponseStatus(HttpStatus.CREATED) @ApiException(START_TASK_GROUP_ERROR) + @OperatorLog(auditType = AuditType.TASK_GROUP_MODIFY) public Result modifyPriority(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "queueId") Integer queueId, @RequestParam(value = "priority") Integer priority) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 2aa2a5eb72..e0055595c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -23,6 +23,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.REMOVE_TASK_INSTANCE_ import static org.apache.dolphinscheduler.api.enums.Status.TASK_SAVEPOINT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.TASK_STOP_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TaskInstanceService; @@ -150,10 +152,12 @@ public class TaskInstanceController extends BaseController { @PostMapping(value = "/{id}/force-success") @ResponseStatus(HttpStatus.OK) @ApiException(FORCE_TASK_SUCCESS_ERROR) - public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @Schema(name = "projectCode", required = true) @PathVariable long projectCode, - @PathVariable(value = "id") Integer id) { - return taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + @OperatorLog(auditType = AuditType.TASK_INSTANCE_FORCE_SUCCESS) + public Result forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @Schema(name = "projectCode", required = true) @PathVariable long projectCode, + @PathVariable(value = "id") Integer id) { + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return Result.success(); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 7ae22f895c..4528024d10 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -24,6 +24,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TENANT_LIST_PAG import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TENANT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_OS_TENANT_CODE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -83,6 +85,7 @@ public class TenantController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_TENANT_ERROR) + @OperatorLog(auditType = AuditType.TENANT_CREATE) public Result createTenant(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "tenantCode") String tenantCode, @RequestParam(value = "queueId") int queueId, @@ -155,6 +158,7 @@ public class TenantController extends BaseController { @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_TENANT_ERROR) + @OperatorLog(auditType = AuditType.TENANT_UPDATE) public Result updateTenant(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id, @RequestParam(value = "tenantCode") String tenantCode, @@ -179,6 +183,7 @@ public class TenantController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_TENANT_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.TENANT_DELETE) public Result deleteTenantById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable(value = "id") int id) throws Exception { tenantService.deleteTenantById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index cc40c6a98c..ac940dde55 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -32,6 +32,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_USER_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.USER_LIST_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_USERNAME_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.UsersService; @@ -88,6 +90,7 @@ public class UsersController extends BaseController { @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_USER_ERROR) + @OperatorLog(auditType = AuditType.USER_CREATE) public Result createUser(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "userName") String userName, @RequestParam(value = "userPassword") String userPassword, @@ -159,6 +162,7 @@ public class UsersController extends BaseController { @PostMapping(value = "/update") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_USER_ERROR) + @OperatorLog(auditType = AuditType.USER_UPDATE) public Result updateUser(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id") int id, @RequestParam(value = "userName") String userName, @@ -196,6 +200,7 @@ public class UsersController extends BaseController { @PostMapping(value = "/delete") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_USER_BY_ID_ERROR) + @OperatorLog(auditType = AuditType.USER_DELETE) public Result delUserById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id") int id) throws Exception { Map result = usersService.deleteUserById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java index 8c332a7bbb..8d025bd873 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java @@ -22,6 +22,8 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_ADDRESS_ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_GROUP_FAIL; import static org.apache.dolphinscheduler.api.enums.Status.SAVE_ERROR; +import org.apache.dolphinscheduler.api.audit.OperatorLog; +import org.apache.dolphinscheduler.api.audit.enums.AuditType; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.WorkerGroupService; import org.apache.dolphinscheduler.api.utils.Result; @@ -80,6 +82,7 @@ public class WorkerGroupController extends BaseController { @PostMapping() @ResponseStatus(HttpStatus.OK) @ApiException(SAVE_ERROR) + @OperatorLog(auditType = AuditType.WORKER_GROUP_CREATE) public Result saveWorkerGroup(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id", required = false, defaultValue = "0") int id, @RequestParam(value = "name") String name, @@ -147,6 +150,7 @@ public class WorkerGroupController extends BaseController { @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_WORKER_GROUP_FAIL) + @OperatorLog(auditType = AuditType.WORKER_GROUP_DELETE) public Result deleteWorkerGroupById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @PathVariable("id") Integer id) { Map result = workerGroupService.deleteWorkerGroupById(loginUser, id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java index 3e3a87681b..ea767f0fa3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2Controller.java @@ -167,8 +167,8 @@ public class TaskInstanceV2Controller extends BaseController { public TaskInstanceSuccessResponse forceTaskSuccess(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, @PathVariable(value = "id") Integer id) { - Result result = taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); - return new TaskInstanceSuccessResponse(result); + taskInstanceService.forceTaskSuccess(loginUser, projectCode, id); + return new TaskInstanceSuccessResponse(Result.success()); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java index 437a2f8b8a..0b36325b44 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/AuditDto.java @@ -19,55 +19,26 @@ package org.apache.dolphinscheduler.api.dto; import java.util.Date; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class AuditDto { private String userName; - private String resource; + private String modelType; + + private String modelName; private String operation; - private Date time; + private Date createTime; - private String resourceName; + private String description; - public String getUserName() { - return userName; - } + private String detail; - public void setUserName(String userName) { - this.userName = userName; - } - - public String getResource() { - return resource; - } - - public void setResource(String resource) { - this.resource = resource; - } - - public String getOperation() { - return operation; - } - - public void setOperation(String operation) { - this.operation = operation; - } - - public Date getTime() { - return time; - } - - public void setTime(Date time) { - this.time = time; - } - - public String getResourceName() { - return resourceName; - } - - public void setResourceName(String resourceName) { - this.resourceName = resourceName; - } + private String latency; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditModelTypeDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditModelTypeDto.java new file mode 100644 index 0000000000..d71836cee4 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditModelTypeDto.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.auditLog; + +import org.apache.dolphinscheduler.common.enums.AuditModelType; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public class AuditModelTypeDto { + + private String name; + + private List child = null; + + public static List getModelTypeDtoList() { + List dtoList = new ArrayList<>(); + transFromEnumListToDto(dtoList, AuditModelType.getAuditModelTreeList()); + return dtoList; + } + + public static List transFromEnumListToDto(List dtoList, + List objectTypeList) { + for (AuditModelType operationType : objectTypeList) { + dtoList.add(transFromEnumToDto(operationType)); + } + + return dtoList; + } + + public static AuditModelTypeDto transFromEnumToDto(AuditModelType operationType) { + AuditModelTypeDto dto = new AuditModelTypeDto(); + dto.setName(operationType.getName()); + + if (!operationType.getChild().isEmpty()) { + dto.setChild(transFromEnumListToDto(new ArrayList<>(), operationType.getChild())); + } + + return dto; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditOperationTypeDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditOperationTypeDto.java new file mode 100644 index 0000000000..2e9f7de365 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/auditLog/AuditOperationTypeDto.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.auditLog; + +import org.apache.dolphinscheduler.common.enums.AuditOperationType; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public class AuditOperationTypeDto { + + private String name; + + public static List getOperationTypeDtoList() { + List dtoList = new ArrayList<>(); + for (AuditOperationType operationType : AuditOperationType.getOperationList()) { + AuditOperationTypeDto dto = new AuditOperationTypeDto(); + dto.setName(operationType.getName()); + dtoList.add(dto); + } + + return dtoList; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java index b5a8ea7a46..d7b026ed16 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskUpdateRequest.java @@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.commons.beanutils.BeanUtils; -import java.lang.reflect.InvocationTargetException; import java.util.Date; import lombok.Data; +import lombok.SneakyThrows; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -107,7 +107,8 @@ public class TaskUpdateRequest { * @param taskDefinition exists task definition object * @return task definition */ - public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) throws InvocationTargetException, IllegalAccessException, InstantiationException, NoSuchMethodException { + @SneakyThrows + public TaskDefinition mergeIntoTaskDefinition(TaskDefinition taskDefinition) { TaskDefinition taskDefinitionDeepCopy = (TaskDefinition) BeanUtils.cloneBean(taskDefinition); assert taskDefinitionDeepCopy != null; if (this.name != null) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java index 149e1abd29..34bd2561b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/recover/RecoverExecuteFunction.java @@ -43,7 +43,7 @@ public class RecoverExecuteFunction implements ExecuteFunction result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value if (project == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -194,7 +194,7 @@ public class PythonGateway { // In the case project exists, but current workflow still not created, we should also return the init // version of it if (processDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); return result; } @@ -202,7 +202,7 @@ public class PythonGateway { TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); if (taskDefinition == null) { - result.put("code", CodeGenerateUtils.getInstance().genCode()); + result.put("code", CodeGenerateUtils.genCode()); result.put("version", 0L); } else { result.put("code", taskDefinition.getCode()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java index 7b998dccce..8fb99eaf73 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AuditService.java @@ -19,9 +19,7 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.dto.AuditDto; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.common.enums.AuditOperationType; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; -import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.AuditLog; /** * audit information service @@ -29,30 +27,28 @@ import org.apache.dolphinscheduler.dao.entity.User; public interface AuditService { /** - * add new audit record + * add audit object * - * @param user login user - * @param resourceType resource type - * @param resourceId resource id - * @param operation operation type + * @param auditLog auditLog */ - void addAudit(User user, AuditResourceType resourceType, Integer resourceId, AuditOperationType operation); + void addAudit(AuditLog auditLog); /** * query audit log list * - * @param loginUser login user - * @param resourceType resource type - * @param operationType operation type - * @param startTime start time - * @param endTime end time - * @param userName query user name - * @param pageNo page number - * @param pageSize page size - * @return audit log string + * @param modelTypes model types + * @param modelName model name + * @param operationTypes operation types + * @param startTime start time + * @param endTime end time + * @param userName query user name + * @param pageNo page number + * @param pageSize page size + * @return audit log string */ - PageInfo queryLogListPaging(User loginUser, AuditResourceType resourceType, - AuditOperationType operationType, String startTime, - String endTime, String userName, + PageInfo queryLogListPaging(String modelTypes, + String operationTypes, String startTime, + String endTime, String userName, String modelName, Integer pageNo, Integer pageSize); + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 86e5396dbe..bff0518685 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -72,9 +72,9 @@ public interface TaskInstanceService { * @param taskInstanceId task instance id * @return the result code and msg */ - Result forceTaskSuccess(User loginUser, - long projectCode, - Integer taskInstanceId); + void forceTaskSuccess(User loginUser, + long projectCode, + Integer taskInstanceId); /** * task savepoint diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index 759e30b036..f8f763b4f4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -17,21 +17,24 @@ package org.apache.dolphinscheduler.api.service.impl; -import org.apache.dolphinscheduler.api.audit.AuditMessage; -import org.apache.dolphinscheduler.api.audit.AuditPublishService; import org.apache.dolphinscheduler.api.dto.AuditDto; import org.apache.dolphinscheduler.api.service.AuditService; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.enums.AuditModelType; import org.apache.dolphinscheduler.common.enums.AuditOperationType; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper; +import org.apache.parquet.Strings; + +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -39,74 +42,69 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @Service +@Slf4j public class AuditServiceImpl extends BaseServiceImpl implements AuditService { @Autowired private AuditLogMapper auditLogMapper; - @Autowired - private AuditPublishService publishService; - - /** - * add new audit log - * - * @param user login user - * @param resourceType resource type - * @param resourceId resource id - * @param operation operation type - */ @Override - public void addAudit(User user, AuditResourceType resourceType, Integer resourceId, AuditOperationType operation) { - publishService.publish(new AuditMessage(user, new Date(), resourceType, operation, resourceId)); + public void addAudit(AuditLog auditLog) { + if (auditLog.getModelId() == null || auditLog.getModelName() == null) { + return; + } + + auditLogMapper.insert(auditLog); } /** * query audit log paging * - * @param loginUser login user - * @param resourceType resource type - * @param operationType operation type - * @param startDate start time - * @param endDate end time - * @param userName query user name - * @param pageNo page number - * @param pageSize page size + * @param modelTypes object types + * @param operationTypes operation types + * @param startDate start time + * @param endDate end time + * @param userName query user name + * @param modelName query object name + * @param pageNo page number + * @param pageSize page size * @return audit log string data */ @Override - public PageInfo queryLogListPaging(User loginUser, - AuditResourceType resourceType, - AuditOperationType operationType, + public PageInfo queryLogListPaging(String modelTypes, + String operationTypes, String startDate, String endDate, String userName, + String modelName, Integer pageNo, Integer pageSize) { - - int[] resourceArray = null; - if (resourceType != null) { - resourceArray = new int[]{resourceType.getCode()}; - } - - int[] opsArray = null; - if (operationType != null) { - opsArray = new int[]{operationType.getCode()}; - } + List objectTypeCodeList = convertStringToList(modelTypes); + List operationTypeCodeList = convertStringToList(operationTypes); Date start = checkAndParseDateParameters(startDate); Date end = checkAndParseDateParameters(endDate); - IPage logIPage = auditLogMapper.queryAuditLog(new Page<>(pageNo, pageSize), resourceArray, opsArray, - userName, start, end); + IPage logIPage = + auditLogMapper.queryAuditLog(new Page<>(pageNo, pageSize), objectTypeCodeList, operationTypeCodeList, + userName, modelName, start, end); List auditDtos = logIPage.getRecords().stream().map(this::transformAuditLog).collect(Collectors.toList()); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - pageInfo.setTotal((int) auditDtos.size()); + pageInfo.setTotal((int) logIPage.getTotal()); pageInfo.setTotalList(auditDtos); return pageInfo; } + private List convertStringToList(String string) { + if (Strings.isNullOrEmpty(string)) { + return new ArrayList<>(); + } + + return Arrays.stream(string.split(",")).collect(Collectors.toList()); + } + /** * transform AuditLog to AuditDto * @@ -115,12 +113,15 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { */ private AuditDto transformAuditLog(AuditLog auditLog) { AuditDto auditDto = new AuditDto(); - String resourceType = AuditResourceType.of(auditLog.getResourceType()).getMsg(); - auditDto.setResource(resourceType); - auditDto.setOperation(AuditOperationType.of(auditLog.getOperation()).getMsg()); + AuditModelType objectType = AuditModelType.of(auditLog.getModelType()); + auditDto.setModelType(objectType.getName()); + auditDto.setModelName(auditLog.getModelName()); + auditDto.setOperation(AuditOperationType.of(auditLog.getOperationType()).getName()); auditDto.setUserName(auditLog.getUserName()); - auditDto.setResourceName(auditLogMapper.queryResourceNameByType(resourceType, auditLog.getResourceId())); - auditDto.setTime(auditLog.getTime()); + auditDto.setLatency(String.valueOf(auditLog.getLatency())); + auditDto.setDetail(auditLog.getDetail()); + auditDto.setDescription(auditLog.getDescription()); + auditDto.setCreateTime(auditLog.getCreateTime()); return auditDto; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index a5ceb92abc..bf4e3ac4d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -96,7 +96,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic cluster.setOperator(loginUser.getId()); cluster.setCreateTime(new Date()); cluster.setUpdateTime(new Date()); - cluster.setCode(CodeGenerateUtils.getInstance().genCode()); + cluster.setCode(CodeGenerateUtils.genCode()); if (clusterMapper.insert(cluster) > 0) { return cluster.getCode(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index ade9aea483..2eb8f84810 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -123,7 +123,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme env.setUpdateTime(new Date()); long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); env.setCode(code); } catch (CodeGenerateException e) { log.error("Generate environment code error.", e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 7ab6102ff8..5b576ce95a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -258,7 +258,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ checkScheduleTimeNumExceed(commandType, cronTime); checkMasterExists(); - long triggerCode = CodeGenerateUtils.getInstance().genCode(); + long triggerCode = CodeGenerateUtils.genCode(); /** * create command diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java index 06f3608207..7543616f31 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java @@ -141,7 +141,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames long code = 0L; try { - code = CodeGenerateUtils.getInstance().genCode(); + code = CodeGenerateUtils.genCode(); cluster.setCode(code); } catch (CodeGenerateUtils.CodeGenerateException e) { log.error("Generate cluster code error.", e); @@ -173,6 +173,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames k8sNamespaceMapper.insert(k8sNamespaceObj); log.info("K8s namespace create complete, namespace:{}.", k8sNamespaceObj.getNamespace()); + result.put(Constants.DATA_LIST, k8sNamespaceObj); putMsg(result, Status.SUCCESS); return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index 0663b88374..c0ecb0e9b5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -237,7 +237,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService host, Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - byte[] logBytes = new byte[0]; + byte[] logBytes; ILogService iLogService = SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(), ILogService.class); @@ -251,6 +251,5 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService log.error("Download TaskInstance: {} Log Error", taskInstance.getName(), ex); throw new ServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR); } - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 09fe59e523..e6893b04fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -238,9 +238,6 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro @Autowired private DataSourceMapper dataSourceMapper; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private WorkFlowLineageService workFlowLineageService; @@ -299,7 +296,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro List taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson); List taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs); - long processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + long processDefinitionCode = CodeGenerateUtils.genCode(); ProcessDefinition processDefinition = new ProcessDefinition(projectCode, name, processDefinitionCode, description, globalParams, locations, timeout, loginUser.getId()); @@ -360,7 +357,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro long processDefinitionCode; try { - processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + processDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -424,7 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson); } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -1233,7 +1230,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // build process definition processDefinition = new ProcessDefinition(projectCode, processDefinitionName, - CodeGenerateUtils.getInstance().genCode(), + CodeGenerateUtils.genCode(), "", "[]", null, 0, loginUser.getId()); @@ -1388,7 +1385,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro sqlParameters.setSqlType(SqlType.NON_QUERY.ordinal()); sqlParameters.setLocalParams(Collections.emptyList()); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); taskDefinition.setTaskType(TASK_TYPE_SQL); taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryInterval(0); @@ -1433,7 +1430,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processDefinition.setProjectCode(projectCode); processDefinition.setUserId(loginUser.getId()); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error( "Save process definition error because generate process definition code error, projectCode:{}.", @@ -1456,7 +1453,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperateTime(now); try { - long code = CodeGenerateUtils.getInstance().genCode(); + long code = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), code); taskDefinitionLog.setCode(code); } catch (CodeGenerateException e) { @@ -1537,6 +1534,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } } + result.put(Constants.DATA_LIST, processDefinition); log.info("Import process definition complete, projectCode:{}, processDefinitionCode:{}.", projectCode, processDefinition.getCode()); return true; @@ -1617,7 +1615,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check whether the process definition json is normal for (TaskNode taskNode : taskNodes) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskNode.getType()) .taskParams(taskNode.getTaskParams()) .dependence(taskNode.getDependence()) @@ -2073,7 +2071,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro Map taskCodeMap = new HashMap<>(); for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { try { - long taskCode = CodeGenerateUtils.getInstance().genCode(); + long taskCode = CodeGenerateUtils.genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), taskCode); taskDefinitionLog.setCode(taskCode); } catch (CodeGenerateException e) { @@ -2096,7 +2094,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } final long oldProcessDefinitionCode = processDefinition.getCode(); try { - processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + processDefinition.setCode(CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error("Generate process definition code error, projectCode:{}.", targetProjectCode, e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 36cc986607..06e9fd95db 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -70,11 +70,9 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.RelationSubWorkflowMapper; -import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; @@ -177,18 +175,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce @Autowired UsersService usersService; - @Autowired - private TenantMapper tenantMapper; - @Autowired TaskDefinitionMapper taskDefinitionMapper; - @Autowired - private TaskPluginManager taskPluginManager; - - @Autowired - private ScheduleMapper scheduleMapper; - @Autowired private RelationSubWorkflowMapper relationSubWorkflowMapper; @@ -725,7 +714,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java index e30375e809..17d1d04712 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java @@ -97,7 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj .builder() .paramName(projectParameterName) .paramValue(projectParameterValue) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .projectCode(projectCode) .userId(loginUser.getId()) .createTime(now) @@ -227,11 +227,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj } for (ProjectParameter projectParameter : projectParameterList) { - try { - this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); - } catch (Exception e) { - throw new ServiceException(Status.DELETE_PROJECT_PARAMETER_ERROR, e.getMessage()); - } + this.deleteProjectParametersByCode(loginUser, projectCode, projectParameter.getCode()); } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java index dcbd3b7456..6274d290d5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java @@ -76,7 +76,7 @@ public class ProjectPreferenceServiceImpl extends BaseServiceImpl projectPreference.setProjectCode(projectCode); projectPreference.setPreferences(preferences); projectPreference.setUserId(loginUser.getId()); - projectPreference.setCode(CodeGenerateUtils.getInstance().genCode()); + projectPreference.setCode(CodeGenerateUtils.genCode()); projectPreference.setState(1); projectPreference.setCreateTime(now); projectPreference.setUpdateTime(now); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 1a00584a26..b5c329c4fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -126,7 +126,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic project = Project .builder() .name(name) - .code(CodeGenerateUtils.getInstance().genCode()) + .code(CodeGenerateUtils.genCode()) .description(desc) .userId(loginUser.getId()) .userName(loginUser.getUserName()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java index 15cbcbb7fa..31dc113dbb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectWorkerGroupMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; @@ -38,6 +39,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -113,23 +115,25 @@ public class ProjectWorkerGroupRelationServiceImpl extends BaseServiceImpl } Set workerGroupNames = - workerGroupMapper.queryAllWorkerGroup().stream().map(item -> item.getName()).collect( + workerGroupMapper.queryAllWorkerGroup().stream().map(WorkerGroup::getName).collect( Collectors.toSet()); workerGroupNames.add(Constants.DEFAULT_WORKER_GROUP); - Set assignedWorkerGroupNames = workerGroups.stream().collect(Collectors.toSet()); + Set assignedWorkerGroupNames = new HashSet<>(workerGroups); Set difference = SetUtils.difference(assignedWorkerGroupNames, workerGroupNames); - if (difference.size() > 0) { + if (!difference.isEmpty()) { putMsg(result, Status.WORKER_GROUP_NOT_EXIST, difference.toString()); return result; } Set projectWorkerGroupNames = projectWorkerGroupMapper.selectList(new QueryWrapper() .lambda() - .eq(ProjectWorkerGroup::getProjectCode, projectCode)).stream().map(item -> item.getWorkerGroup()) + .eq(ProjectWorkerGroup::getProjectCode, projectCode)) + .stream() + .map(ProjectWorkerGroup::getWorkerGroup) .collect(Collectors.toSet()); difference = SetUtils.difference(projectWorkerGroupNames, assignedWorkerGroupNames); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 6a15da17a8..1c039cdfbd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkFilePath; import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS; import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; @@ -1290,6 +1291,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { return; } + // abnormal characters check + if (!checkFilePath(fullName)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH); + } // Avoid returning to the parent directory if (fullName.contains("../")) { throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java index 80e59670f7..6967222b7f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java @@ -808,6 +808,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe log.info("Schedule update complete, projectCode:{}, processDefinitionCode:{}, scheduleId:{}.", processDefinition.getProjectCode(), processDefinition.getCode(), schedule.getId()); + result.put(Constants.DATA_LIST, schedule); putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 4a0c3f8b68..2887606a1d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -75,7 +75,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -105,8 +104,6 @@ import com.google.common.collect.Lists; @Slf4j public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDefinitionService { - private static final String RELEASESTATE = "releaseState"; - @Autowired private ProjectMapper projectMapper; @@ -137,9 +134,6 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe @Autowired private ProcessService processService; - @Autowired - private TaskPluginManager taskPluginManager; - @Autowired private ProcessDefinitionService processDefinitionService; @@ -149,8 +143,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create task definition * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param taskDefinitionJson task definition json */ @Transactional @@ -173,7 +167,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionLog.getTaskType()) .taskParams(taskDefinitionLog.getTaskParams()) .dependence(taskDefinitionLog.getDependence()) @@ -214,7 +208,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe Project project = projectMapper.queryByCode(taskDefinition.getProjectCode()); projectService.checkProjectAndAuthThrowException(user, project, permissions); - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -244,7 +238,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Create resource task definition * - * @param loginUser login user + * @param loginUser login user * @param taskCreateRequest task definition json * @return new TaskDefinition have created */ @@ -265,7 +259,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe long taskDefinitionCode; try { - taskDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + taskDefinitionCode = CodeGenerateUtils.genCode(); } catch (CodeGenerateException e) { throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } @@ -288,11 +282,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * create single task definition that binds the workflow * - * @param loginUser login user - * @param projectCode project code + * @param loginUser login user + * @param projectCode project code * @param processDefinitionCode process definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return create result code */ @Transactional @@ -327,7 +321,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return result; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinition.getTaskType()) .taskParams(taskDefinition.getTaskParams()) .dependence(taskDefinition.getDependence()) @@ -338,7 +332,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe } long taskCode = taskDefinition.getCode(); if (taskCode == 0) { - taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + taskDefinition.setCode(CodeGenerateUtils.genCode()); } List processTaskRelationLogList = processTaskRelationMapper.queryByProcessCode(processDefinitionCode) @@ -414,10 +408,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * query task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code * @param processCode process code - * @param taskName task name + * @param taskName task name */ @Override public Map queryTaskDefinitionByName(User loginUser, long projectCode, long processCode, @@ -475,12 +469,12 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Delete resource task definition by code - * + *

* Only task release state offline and no downstream tasks can be deleted, will also remove the exists * task relation [upstreamTaskCode, taskCode] * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code */ @Transactional @Override @@ -548,9 +542,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task code * @param taskDefinitionJsonObj task definition json object */ @Transactional @@ -606,8 +600,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition * - * @param loginUser login user - * @param taskCode task code + * @param loginUser login user + * @param taskCode task code * @param taskUpdateRequest task definition json object * @return new TaskDefinition have updated */ @@ -621,13 +615,8 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe throw new ServiceException(Status.TASK_DEFINITION_NOT_EXISTS, taskCode); } - TaskDefinition taskDefinitionUpdate; - try { - taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); - } catch (InvocationTargetException | IllegalAccessException | InstantiationException - | NoSuchMethodException e) { - throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, taskUpdateRequest.toString()); - } + TaskDefinition taskDefinitionUpdate = taskUpdateRequest.mergeIntoTaskDefinition(taskDefinitionOriginal); + this.checkTaskDefinitionValid(loginUser, taskDefinitionUpdate, TASK_DEFINITION_UPDATE); this.TaskDefinitionUpdateValid(taskDefinitionOriginal, taskDefinitionUpdate); @@ -658,7 +647,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe * Get resource task definition by code * * @param loginUser login user - * @param taskCode task code + * @param taskCode task code * @return TaskDefinition */ @Override @@ -676,7 +665,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * Get resource task definition according to query parameter * - * @param loginUser login user + * @param loginUser login user * @param taskFilterRequest taskFilterRequest object you want to filter the resource task definitions * @return TaskDefinitions of page */ @@ -743,7 +732,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return null; } - if (!taskPluginManager.checkTaskParameters(ParametersNode.builder() + if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() .taskType(taskDefinitionToUpdate.getTaskType()) .taskParams(taskDefinitionToUpdate.getTaskParams()) .dependence(taskDefinitionToUpdate.getDependence()) @@ -848,11 +837,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * update task definition and upstream * - * @param loginUser login user - * @param projectCode project code - * @param taskCode task definition code + * @param loginUser login user + * @param projectCode project code + * @param taskCode task definition code * @param taskDefinitionJsonObj task definition json object - * @param upstreamCodes upstream task codes, sep comma + * @param upstreamCodes upstream task codes, sep comma * @return update result code */ @Override @@ -1021,10 +1010,10 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * switch task definition * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code - * @param taskCode task code - * @param version the version user want to switch + * @param taskCode task code + * @param version the version user want to switch */ @Transactional @Override @@ -1264,7 +1253,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe List taskCodes = new ArrayList<>(); try { for (int i = 0; i < genNum; i++) { - taskCodes.add(CodeGenerateUtils.getInstance().genCode()); + taskCodes.add(CodeGenerateUtils.genCode()); } } catch (CodeGenerateException e) { log.error("Generate task definition code error.", e); @@ -1279,9 +1268,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe /** * release task definition * - * @param loginUser login user - * @param projectCode project code - * @param code task definition code + * @param loginUser login user + * @param projectCode project code + * @param code task definition code * @param releaseState releaseState * @return update result code */ @@ -1297,7 +1286,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } if (null == releaseState) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(code); @@ -1337,7 +1326,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe break; default: log.warn("Parameter releaseState is invalid."); - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.RELEASE_STATE); return result; } int update = taskDefinitionMapper.updateById(taskDefinition); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java index a1d9b4acf8..379c629ea6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java @@ -129,6 +129,7 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe if (taskGroupMapper.insert(taskGroup) > 0) { log.info("Create task group complete, taskGroupName:{}.", taskGroup.getName()); + result.put(Constants.DATA_LIST, taskGroup); putMsg(result, Status.SUCCESS); } else { log.error("Create task group error, taskGroupName:{}.", taskGroup.getName()); @@ -194,6 +195,7 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe int i = taskGroupMapper.updateById(taskGroup); if (i > 0) { log.info("Update task group complete, taskGroupId:{}.", id); + result.put(Constants.DATA_LIST, taskGroup); putMsg(result, Status.SUCCESS); } else { log.error("Update task group error, taskGroupId:{}.", id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index f06f8115a9..7469d8db13 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -23,6 +23,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.service.TaskGroupQueueService; import org.apache.dolphinscheduler.api.service.TaskInstanceService; @@ -33,14 +34,15 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.repository.DqExecuteResultDao; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils; import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; @@ -107,6 +109,9 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst @Autowired private TaskGroupQueueService taskGroupQueueService; + @Autowired + private ProcessInstanceDao workflowInstanceDao; + /** * query task list by project, process instance, task name, task start time, task end time, task status, keyword paging * @@ -216,58 +221,39 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst */ @Transactional @Override - public Result forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { - Result result = new Result(); - Project project = projectMapper.queryByCode(projectCode); + public void forceTaskSuccess(User loginUser, long projectCode, Integer taskInstanceId) { // check user access for project - Map checkResult = - projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); - Status status = (Status) checkResult.get(Constants.STATUS); - if (status != Status.SUCCESS) { - putMsg(result, status); - return result; + projectService.checkProjectAndAuthThrowException(loginUser, projectCode, FORCED_SUCCESS); + + TaskInstance task = taskInstanceDao.queryOptionalById(taskInstanceId) + .orElseThrow(() -> new ServiceException(Status.TASK_INSTANCE_NOT_FOUND)); + + if (task.getProjectCode() != projectCode) { + throw new ServiceException("The task instance is not under the project: " + projectCode); } - // check whether the task instance can be found - TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); - if (task == null) { - log.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, - taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); - return result; - } - - TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(task.getTaskCode()); - if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - log.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, - task.getTaskCode()); - putMsg(result, Status.TASK_INSTANCE_NOT_FOUND, taskInstanceId); - return result; + ProcessInstance processInstance = workflowInstanceDao.queryOptionalById(task.getProcessInstanceId()) + .orElseThrow( + () -> new ServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, task.getProcessInstanceId())); + if (!processInstance.getState().isFinished()) { + throw new ServiceException("The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance"); } // check whether the task instance state type is failure or cancel if (!task.getState().isFailure() && !task.getState().isKill()) { - log.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", - task.getState().getDesc(), projectCode, taskInstanceId); - putMsg(result, Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState().toString()); - return result; + throw new ServiceException(Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState()); } // change the state of the task instance task.setState(TaskExecutionStatus.FORCED_SUCCESS); task.setEndTime(new Date()); int changedNum = taskInstanceMapper.updateById(task); - if (changedNum > 0) { - processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId); - log.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, - taskInstanceId); - putMsg(result, Status.SUCCESS); - } else { - log.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", - projectCode, taskInstanceId); - putMsg(result, Status.FORCE_TASK_SUCCESS_ERROR); + if (changedNum <= 0) { + throw new ServiceException(Status.FORCE_TASK_SUCCESS_ERROR); } - return result; + processService.forceProcessInstanceSuccessByTaskInstanceId(task); + log.info("Force success task instance:{} success", taskInstanceId); } @Override diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index 88cd3247ae..9c98880740 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -157,6 +157,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro handleDefaultWorkGroup(workerGroupMapper, workerGroup, loginUser, otherParamsJson); log.info("Worker group save complete, workerGroupName:{}.", workerGroup.getName()); putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, workerGroup); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 8b166a16dd..b394d4956c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -158,4 +158,14 @@ public class CheckUtils { return pattern.matcher(str).matches(); } + + /** + * regex FilePath check,only use a to z, A to Z, 0 to 9, and _./- + * + * @param str input string + * @return true if regex pattern is right, otherwise return false + */ + public static boolean checkFilePath(String str) { + return regexChecks(str, Constants.REGEX_FILE_PATH); + } } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index c93fd99a59..e38d0c5a8e 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -51,15 +51,7 @@ spring: password: root hikari: connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 quartz: auto-startup: false job-store-type: jdbc @@ -149,8 +141,8 @@ api: #tenant1: 11 #tenant2: 20 python-gateway: - # Weather enable python gateway server or not. The default value is true. - enabled: true + # Weather enable python gateway server or not. The default value is false. + enabled: false # Authentication token for connection from python api to python gateway server. Should be changed the default value # when you deploy in public network. auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java index d2da5bc638..eae064bb24 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java @@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import java.text.MessageFormat; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; @@ -30,6 +32,12 @@ public class AssertionsHelper extends Assertions { Assertions.assertEquals(status.getCode(), exception.getCode()); } + public static void assertThrowsServiceException(String message, Executable executable) { + ServiceException exception = Assertions.assertThrows(ServiceException.class, executable); + Assertions.assertEquals(MessageFormat.format(Status.INTERNAL_SERVER_ERROR_ARGS.getMsg(), message), + exception.getMessage()); + } + public static void assertDoesNotThrow(Executable executable) { Assertions.assertDoesNotThrow(executable); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberTest.java deleted file mode 100644 index fe8e5303f5..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/audit/AuditSubscriberTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.audit; - -import org.apache.dolphinscheduler.common.enums.AuditOperationType; -import org.apache.dolphinscheduler.common.enums.AuditResourceType; -import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper; - -import java.util.Date; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class AuditSubscriberTest { - - @Mock - private AuditLogMapper logMapper; - - @InjectMocks - private AuditSubscriberImpl auditSubscriber; - - @Test - public void testExecute() { - Mockito.when(logMapper.insert(Mockito.any(AuditLog.class))).thenReturn(1); - auditSubscriber.execute( - new AuditMessage(new User(), new Date(), AuditResourceType.USER_MODULE, AuditOperationType.CREATE, 1)); - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java index 5d8308760b..9114daa04f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java @@ -47,7 +47,7 @@ public class AccessTokenControllerTest extends AbstractControllerTest { @Test public void testCreateToken() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "4"); + paramsMap.add("userId", "1"); paramsMap.add("expireTime", "2019-12-18 00:00:00"); paramsMap.add("token", "607f5aeaaa2093dbdff5d5522ce00510"); MvcResult mvcResult = mockMvc.perform(post("/access-tokens") @@ -64,7 +64,7 @@ public class AccessTokenControllerTest extends AbstractControllerTest { @Test public void testCreateTokenIfAbsent() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "4"); + paramsMap.add("userId", "1"); paramsMap.add("expireTime", "2019-12-18 00:00:00"); paramsMap.add("token", null); @@ -101,7 +101,7 @@ public class AccessTokenControllerTest extends AbstractControllerTest { @Test public void testGenerateToken() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "4"); + paramsMap.add("userId", "1"); paramsMap.add("expireTime", "2019-12-28 00:00:00"); MvcResult mvcResult = mockMvc.perform(post("/access-tokens/generate") .header("sessionId", sessionId) @@ -161,7 +161,7 @@ public class AccessTokenControllerTest extends AbstractControllerTest { public void testUpdateToken() throws Exception { testCreateToken(); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "4"); + paramsMap.add("userId", "1"); paramsMap.add("expireTime", "2019-12-20 00:00:00"); paramsMap.add("token", "cxctoken123update"); MvcResult mvcResult = mockMvc.perform(put("/access-tokens/1") @@ -180,7 +180,7 @@ public class AccessTokenControllerTest extends AbstractControllerTest { this.testCreateTokenIfAbsent(); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "4"); + paramsMap.add("userId", "1"); paramsMap.add("expireTime", "2019-12-20 00:00:00"); paramsMap.add("token", null); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceControllerTest.java index 5ad7b2da52..649d95bb4c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceControllerTest.java @@ -35,8 +35,10 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AlertPluginInstanceType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.User; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.MediaType; @@ -49,6 +51,7 @@ import org.springframework.util.MultiValueMap; */ public class AlertPluginInstanceControllerTest extends AbstractControllerTest { + private static AlertPluginInstance alertPluginInstance = new AlertPluginInstance(); private static final int pluginDefineId = 1; private static final String instanceName = "instanceName"; private static final String pluginInstanceParams = "pluginInstanceParams"; @@ -60,6 +63,12 @@ public class AlertPluginInstanceControllerTest extends AbstractControllerTest { @MockBean(name = "alertPluginInstanceServiceImpl") private AlertPluginInstanceService alertPluginInstanceService; + @BeforeAll + public static void initInstance() { + alertPluginInstance.setId(1); + alertPluginInstance.setInstanceName(instanceName); + } + @Test public void testCreateAlertPluginInstance() throws Exception { // Given @@ -71,7 +80,7 @@ public class AlertPluginInstanceControllerTest extends AbstractControllerTest { paramsMap.add("pluginInstanceParams", pluginInstanceParams); when(alertPluginInstanceService.create(any(User.class), eq(pluginDefineId), eq(instanceName), - eq(pluginInstanceType), eq(warningType), eq(pluginInstanceParams))).thenReturn(null); + eq(pluginInstanceType), eq(warningType), eq(pluginInstanceParams))).thenReturn(alertPluginInstance); // When final MvcResult mvcResult = mockMvc.perform(post("/alert-plugin-instances") @@ -122,7 +131,7 @@ public class AlertPluginInstanceControllerTest extends AbstractControllerTest { paramsMap.add("pluginInstanceParams", pluginInstanceParams); when(alertPluginInstanceService.updateById(any(User.class), eq(pluginDefineId), eq(instanceName), - eq(warningType), eq(pluginInstanceParams))).thenReturn(null); + eq(warningType), eq(pluginInstanceParams))).thenReturn(alertPluginInstance); // When final MvcResult mvcResult = mockMvc.perform(put("/alert-plugin-instances/{id}", pluginDefineId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java index 7cd97a5551..2c1a62ccc6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecuteFunctionControllerTest.java @@ -80,7 +80,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { final int dryRun = 7; final int testFlag = 0; final ComplementDependentMode complementDependentMode = ComplementDependentMode.OFF_MODE; - final Integer version = null; + final Integer version = 1; final boolean allLevelDependent = false; final JsonObject expectResponseContent = gson .fromJson("{\"code\":0,\"msg\":\"success\",\"data\":\"Test Data\",\"success\":true,\"failed\":false}", @@ -115,6 +115,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { paramsMap.add("dryRun", String.valueOf(dryRun)); paramsMap.add("testFlag", String.valueOf(testFlag)); paramsMap.add("executionOrder", String.valueOf(executionOrder)); + paramsMap.add("version", String.valueOf(version)); when(executorService.execProcessInstance(any(User.class), eq(projectCode), eq(processDefinitionCode), eq(scheduleTime), eq(execType), eq(failureStrategy), eq(startNodeList), eq(taskDependType), @@ -162,6 +163,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { paramsMap.add("dryRun", String.valueOf(dryRun)); paramsMap.add("testFlag", String.valueOf(testFlag)); paramsMap.add("executionOrder", String.valueOf(executionOrder)); + paramsMap.add("version", String.valueOf(version)); when(executorService.execProcessInstance(any(User.class), eq(projectCode), eq(processDefinitionCode), eq(scheduleTime), eq(execType), eq(failureStrategy), eq(startNodeList), eq(taskDependType), @@ -209,6 +211,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { paramsMap.add("dryRun", String.valueOf(dryRun)); paramsMap.add("testFlag", String.valueOf(testFlag)); paramsMap.add("executionOrder", String.valueOf(executionOrder)); + paramsMap.add("version", String.valueOf(version)); when(executorService.execProcessInstance(any(User.class), eq(projectCode), eq(processDefinitionCode), eq(scheduleTime), eq(execType), eq(failureStrategy), eq(startNodeList), eq(taskDependType), @@ -241,6 +244,7 @@ public class ExecuteFunctionControllerTest extends AbstractControllerTest { paramsMap.add("failureStrategy", String.valueOf(failureStrategy)); paramsMap.add("warningType", String.valueOf(warningType)); paramsMap.add("scheduleTime", scheduleTime); + paramsMap.add("version", String.valueOf(version)); when(executorService.execProcessInstance(any(User.class), eq(projectCode), eq(processDefinitionCode), eq(scheduleTime), eq(null), eq(failureStrategy), eq(null), eq(null), eq(warningType), diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java index 5644065a33..1be311fbaf 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.SchedulerService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.WarningType; @@ -38,6 +39,7 @@ import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.User; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; @@ -48,13 +50,25 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import com.google.common.collect.ImmutableMap; + public class SchedulerControllerTest extends AbstractControllerTest { + private static Schedule scheduleObj = new Schedule(); + private static final Logger logger = LoggerFactory.getLogger(SchedulerControllerTest.class); + final ImmutableMap result = + ImmutableMap.of(Constants.STATUS, Status.SUCCESS, Constants.DATA_LIST, scheduleObj); + @MockBean(name = "schedulerService") private SchedulerService schedulerService; + @BeforeAll + public static void initInstance() { + scheduleObj.setId(1); + } + @Test public void testCreateSchedule() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -72,7 +86,7 @@ public class SchedulerControllerTest extends AbstractControllerTest { Mockito.when(schedulerService.insertSchedule(isA(User.class), isA(Long.class), isA(Long.class), isA(String.class), isA(WarningType.class), isA(int.class), isA(FailureStrategy.class), - isA(Priority.class), isA(String.class), isA(String.class), isA(Long.class))).thenReturn(success()); + isA(Priority.class), isA(String.class), isA(String.class), isA(Long.class))).thenReturn(result); MvcResult mvcResult = mockMvc.perform(post("/projects/{projectCode}/schedules/", 123) .header(SESSION_ID, sessionId) @@ -103,7 +117,7 @@ public class SchedulerControllerTest extends AbstractControllerTest { Mockito.when(schedulerService.updateSchedule(isA(User.class), isA(Long.class), isA(Integer.class), isA(String.class), isA(WarningType.class), isA(Integer.class), isA(FailureStrategy.class), - isA(Priority.class), isA(String.class), isA(String.class), isA(Long.class))).thenReturn(success()); + isA(Priority.class), isA(String.class), isA(String.class), isA(Long.class))).thenReturn(result); MvcResult mvcResult = mockMvc.perform(put("/projects/{projectCode}/schedules/{id}", 123, 37) .header(SESSION_ID, sessionId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 7ebe5bf757..b58944537b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -82,9 +82,7 @@ public class TaskInstanceControllerTest extends AbstractControllerTest { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("taskInstanceId", "104"); - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); - when(taskInstanceService.forceTaskSuccess(any(User.class), anyLong(), anyInt())).thenReturn(mockResult); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(User.class), anyLong(), anyInt()); MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/task-instance/force-success", "cxc_1113") .header(SESSION_ID, sessionId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java index cc70f200e1..8e76ec1694 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/TaskInstanceV2ControllerTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.controller.AbstractControllerTest; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceQueryRequest; +import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceSuccessResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -85,12 +86,9 @@ public class TaskInstanceV2ControllerTest extends AbstractControllerTest { @Test public void testForceTaskSuccess() { - Result mockResult = new Result(); - putMsg(mockResult, Status.SUCCESS); + Mockito.doNothing().when(taskInstanceService).forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt()); - when(taskInstanceService.forceTaskSuccess(any(), Mockito.anyLong(), Mockito.anyInt())).thenReturn(mockResult); - - Result taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); + TaskInstanceSuccessResponse taskResult = taskInstanceV2Controller.forceTaskSuccess(null, 1L, 1); Assertions.assertEquals(Integer.valueOf(Status.SUCCESS.getCode()), taskResult.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AuditServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AuditServiceTest.java index 3bde4a1b8d..86238f0598 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AuditServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AuditServiceTest.java @@ -21,9 +21,10 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.service.impl.AuditServiceImpl; +import org.apache.dolphinscheduler.common.enums.AuditModelType; +import org.apache.dolphinscheduler.common.enums.AuditOperationType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.AuditLog; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper; import java.util.ArrayList; @@ -66,15 +67,16 @@ public class AuditServiceTest { page.setRecords(getLists()); page.setTotal(1L); when(auditLogMapper.queryAuditLog(Mockito.any(Page.class), Mockito.any(), Mockito.any(), Mockito.eq(""), + Mockito.eq(""), eq(start), eq(end))).thenReturn(page); Assertions.assertDoesNotThrow(() -> { auditService.queryLogListPaging( - new User(), - null, - null, + "", + "", "2020-11-01 00:00:00", "2020-11-02 00:00:00", "", + "", 1, 10); }); @@ -89,8 +91,8 @@ public class AuditServiceTest { private AuditLog getAuditLog() { AuditLog auditLog = new AuditLog(); auditLog.setUserName("testName"); - auditLog.setOperation(0); - auditLog.setResourceType(0); + auditLog.setOperationType(AuditOperationType.CREATE.getName()); + auditLog.setModelType(AuditModelType.PROJECT.getName()); return auditLog; } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 2c4de2ab7e..4861e1004e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -18,8 +18,10 @@ package org.apache.dolphinscheduler.api.service; import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; @@ -109,11 +111,25 @@ public class LoggerServiceTest { @Override public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { - return new TaskInstanceLogFileDownloadResponse(new byte[0]); + if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 1) { + return new TaskInstanceLogFileDownloadResponse(new byte[0]); + } else if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogFileDownloadResponse("log content".getBytes()); + } + + throw new ServiceException("download error"); } @Override public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() != null) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogPageQueryResponse("log content"); + } + } + return new TaskInstanceLogPageQueryResponse(); } @@ -177,6 +193,13 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); result = loggerService.queryLog(loginUser, 1, 1, 1); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + result = loggerService.queryLog(loginUser, 1, 0, 1); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + taskInstance.setLogPath(""); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, 1, 1, 1)); } @Test @@ -237,9 +260,15 @@ public class LoggerServiceTest { loginUser.setUserType(UserType.GENERAL_USER); TaskInstance taskInstance = new TaskInstance(); when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); + when(taskInstanceDao.queryById(10)).thenReturn(null); + + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); + TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(projectCode); taskDefinition.setCode(1L); + // SUCCESS taskInstance.setTaskCode(1L); taskInstance.setId(1); @@ -249,13 +278,27 @@ public class LoggerServiceTest { when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(10); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> loggerService.queryLog(loginUser, projectCode, 1, 1, 1)); + + taskDefinition.setProjectCode(1); + taskInstance.setId(10); + when(taskInstanceDao.queryById(10)).thenReturn(taskInstance); + String result = loggerService.queryLog(loginUser, projectCode, 10, 1, 1); + assertEquals("log content", result); + + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.QUERY_TASK_INSTANCE_LOG_ERROR, + () -> loggerService.queryLog(loginUser, projectCode, 10, 1, 1)); } @Test public void testGetLogBytesInSpecifiedProject() { long projectCode = 1L; when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Project project = getProject(projectCode); User loginUser = new User(); loginUser.setId(-1); @@ -272,9 +315,24 @@ public class LoggerServiceTest { taskInstance.setHost("127.0.0.1:" + nettyServerPort); taskInstance.setLogPath("/temp/log"); doNothing().when(projectService).checkProjectAndAuthThrowException(loginUser, projectCode, DOWNLOAD_LOG); + + when(taskInstanceDao.queryById(1)).thenReturn(null); + assertThrowsServiceException( + Status.INTERNAL_SERVER_ERROR_ARGS, () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); assertDoesNotThrow(() -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(2L); + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> loggerService.getLogBytes(loginUser, projectCode, 1)); + + taskDefinition.setProjectCode(1L); + taskInstance.setId(100); + when(taskInstanceDao.queryById(100)).thenReturn(taskInstance); + assertThrowsServiceException(Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR, + () -> loggerService.getLogBytes(loginUser, projectCode, 100)); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 9fb4d83052..7d9a4f9338 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -84,6 +84,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -142,9 +143,6 @@ public class ProcessInstanceServiceTest { @Mock TaskDefinitionMapper taskDefinitionMapper; - @Mock - TaskPluginManager taskPluginManager; - @Mock ScheduleMapper scheduleMapper; @@ -625,21 +623,27 @@ public class ProcessInstanceServiceTest { List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result); putMsg(result, Status.SUCCESS, projectCode); - when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Map processInstanceFinishRes = - processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - // success - when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); - putMsg(result, Status.SUCCESS, projectCode); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Map processInstanceFinishRes = + processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", true, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); - when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) - .thenReturn(1); - Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); - Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + // success + when(processDefineMapper.queryByCode(46L)).thenReturn(processDefinition); + putMsg(result, Status.SUCCESS, projectCode); + + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)) + .thenReturn(1); + Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, + taskRelationJson, taskDefinitionJson, "2020-02-21 00:00:00", Boolean.FALSE, "", "", 0); + Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java index 7a3fb1b68d..ca51690ce6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectParameterServiceTest.java @@ -17,27 +17,40 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectParameterServiceImpl; import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectParameter; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; -import org.junit.jupiter.api.Assertions; +import java.util.Collections; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class ProjectParameterServiceTest { @@ -60,92 +73,187 @@ public class ProjectParameterServiceTest { public void testCreateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); + // PERMISSION DENIED + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + + // CODE GENERATION ERROR + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); + } + + // PROJECT_PARAMETER_ALREADY_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // INSERT DATA ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(-1); + result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); + assertEquals(Status.CREATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); + when(projectParameterMapper.insert(Mockito.any())).thenReturn(1); result = projectParameterService.createProjectParameter(loginUser, projectCode, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testUpdateProjectParameter() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // PROJECT_PARAMETER_ALREADY_EXISTS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(getProjectParameter()); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key", "value"); - Assertions.assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + assertEquals(Status.PROJECT_PARAMETER_ALREADY_EXISTS.getCode(), result.getCode()); + + // PROJECT_UPDATE_ERROR + when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); + assertEquals(Status.UPDATE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.selectOne(Mockito.any())).thenReturn(null); - Mockito.when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); + when(projectParameterMapper.updateById(Mockito.any())).thenReturn(1); result = projectParameterService.updateProjectParameter(loginUser, projectCode, 1, "key1", "value"); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); ProjectParameter projectParameter = (ProjectParameter) result.getData(); - Assertions.assertNotNull(projectParameter.getOperator()); - Assertions.assertNotNull(projectParameter.getUpdateTime()); + assertNotNull(projectParameter.getOperator()); + assertNotNull(projectParameter.getUpdateTime()); } @Test public void testDeleteProjectParametersByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) - .thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); Result result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + + // DATABASE OPERATION ERROR + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(-1); + result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); + assertEquals(Status.DELETE_PROJECT_PARAMETER_ERROR.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); - Mockito.when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); + when(projectParameterMapper.deleteById(Mockito.anyInt())).thenReturn(1); result = projectParameterService.deleteProjectParametersByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } @Test public void testQueryProjectParameterByCode() { User loginUser = getGeneralUser(); - // PROJECT_PARAMETER_NOT_EXISTS - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), - Mockito.any())).thenReturn(true); - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + Result result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // PROJECT_PARAMETER_NOT_EXISTS + when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())).thenReturn(true); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(null); + result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); + assertEquals(Status.PROJECT_PARAMETER_NOT_EXISTS.getCode(), result.getCode()); // SUCCESS - Mockito.when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); + when(projectParameterMapper.queryByCode(Mockito.anyLong())).thenReturn(getProjectParameter()); result = projectParameterService.queryProjectParameterByCode(loginUser, projectCode, 1); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + @Test + public void testQueryProjectParameterListPaging() { + User loginUser = getGeneralUser(); + Integer pageSize = 10; + Integer pageNo = 1; + + // NO PERMISSION + when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), + Mockito.any())) + .thenReturn(false); + + Result result = + projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, null); + assertNull(result.getData()); + assertNull(result.getCode()); + assertNull(result.getMsg()); + + // SUCCESS + when(projectService.hasProjectAndPerm(any(), any(), any(Result.class), any())) + .thenReturn(true); + + Page page = new Page<>(pageNo, pageSize); + page.setRecords(Collections.singletonList(getProjectParameter())); + + when(projectParameterMapper.queryProjectParameterListPaging(any(), anyLong(), any(), any())).thenReturn(page); + result = projectParameterService.queryProjectParameterListPaging(loginUser, projectCode, pageSize, pageNo, + null); + assertEquals(Status.SUCCESS.getCode(), result.getCode()); + } + + @Test + public void testBatchDeleteProjectParametersByCodes() { + User loginUser = getGeneralUser(); + + Result result = projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, ""); + assertEquals(Status.PROJECT_PARAMETER_CODE_EMPTY.getCode(), result.getCode()); + + when(projectParameterMapper.queryByCodes(any())).thenReturn(Collections.singletonList(getProjectParameter())); + + AssertionsHelper.assertThrowsServiceException(Status.PROJECT_PARAMETER_NOT_EXISTS, + () -> projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1,2")); + + projectParameterService.batchDeleteProjectParametersByCodes(loginUser, projectCode, "1"); } private Project getProject(long projectCode) { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java index 530c15d48e..7a74a7c265 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectPreferenceServiceTest.java @@ -60,28 +60,65 @@ public class ProjectPreferenceServiceTest { public void testUpdateProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); + + // success Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); - Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(1); - Result result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operatation fail + Mockito.when(projectPreferenceMapper.insert(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.CREATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); + + // when preference exists in project + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + + // success + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // database operation fail + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.updateProjectPreference(loginUser, projectCode, "value"); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_ERROR.getCode(), result.getCode()); } @Test public void testQueryProjectPreferenceByProjectCode() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + // PROJECT_PARAMETER_NOT_EXISTS Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class), Mockito.any())).thenReturn(true); Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(null); - Result result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); + result = projectPreferenceService.queryProjectPreferenceByProjectCode(loginUser, projectCode); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); // SUCCESS @@ -94,14 +131,29 @@ public class ProjectPreferenceServiceTest { public void testEnableProjectPreference() { User loginUser = getGeneralUser(); + // no permission + Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) + .thenReturn(false); + Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Assertions.assertNull(result.getCode()); + Assertions.assertNull(result.getData()); + Assertions.assertNull(result.getMsg()); + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); Mockito.when(projectService.hasProjectAndWritePerm(Mockito.any(), Mockito.any(), Mockito.any(Result.class))) .thenReturn(true); + // success Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); - Result result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 1); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + // db operation fail + Mockito.when(projectPreferenceMapper.selectOne(Mockito.any())).thenReturn(getProjectPreference()); + Mockito.when(projectPreferenceMapper.updateById(Mockito.any())).thenReturn(-1); + result = projectPreferenceService.enableProjectPreference(loginUser, projectCode, 2); + Assertions.assertEquals(Status.UPDATE_PROJECT_PREFERENCE_STATE_ERROR.getCode(), result.getCode()); } private User getGeneralUser() { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java index a8e8b8beb2..6f6b7ca2d6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java @@ -17,13 +17,17 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getAdminUser; +import static org.apache.dolphinscheduler.api.utils.ServiceTestUtil.getGeneralUser; + +import org.apache.dolphinscheduler.api.AssertionsHelper; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProjectWorkerGroupRelationServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectWorkerGroup; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -33,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -77,27 +82,87 @@ public class ProjectWorkerGroupRelationServiceTest { @Test public void testAssignWorkerGroupsToProject() { + User generalUser = getGeneralUser(); User loginUser = getAdminUser(); - Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); - Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + // no permission + Result result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(generalUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), result.getCode()); + + // project code is null + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, null, getWorkerGroups()); Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + // worker group is empty + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + Collections.emptyList()); + Assertions.assertEquals(Status.WORKER_GROUP_TO_PROJECT_IS_EMPTY.getCode(), result.getCode()); + + // project not exists + Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(null); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), result.getCode()); + + // worker group not exists WorkerGroup workerGroup = new WorkerGroup(); workerGroup.setName("test"); Mockito.when(projectMapper.queryByCode(Mockito.anyLong())).thenReturn(getProject()); - Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Lists.newArrayList(workerGroup)); + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getDiffWorkerGroups()); + Assertions.assertEquals(Status.WORKER_GROUP_NOT_EXIST.getCode(), result.getCode()); + + // db insertion fail + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(Collections.singletonList(workerGroup)); + Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // success Mockito.when(projectWorkerGroupMapper.insert(Mockito.any())).thenReturn(1); result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, getWorkerGroups()); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // success when there is diff between current wg and assigned wg + Mockito.when(projectWorkerGroupMapper.selectList(Mockito.any())) + .thenReturn(Collections.singletonList(getDiffProjectWorkerGroup())); + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(1); + result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups()); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode()); + + // db deletion fail + Mockito.when(projectWorkerGroupMapper.delete(Mockito.any())).thenReturn(-1); + AssertionsHelper.assertThrowsServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); + + // fail when wg is referenced by task definition + Mockito.when(taskDefinitionMapper.queryAllDefinitionList(Mockito.anyLong())) + .thenReturn(Collections.singletonList(getTaskDefinitionWithDiffWorkerGroup())); + AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS, + () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode, + getWorkerGroups())); } @Test public void testQueryWorkerGroupsByProject() { + // no permission + Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) + .thenReturn(false); + Map result = + projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + + Assertions.assertTrue(result.isEmpty()); + + // success Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any())) .thenReturn(true); @@ -113,8 +178,7 @@ public class ProjectWorkerGroupRelationServiceTest { Mockito.when(scheduleMapper.querySchedulerListByProjectName(Mockito.any())) .thenReturn(Lists.newArrayList()); - Map result = - projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); + result = projectWorkerGroupRelationService.queryWorkerGroupsByProject(getGeneralUser(), projectCode); ProjectWorkerGroup[] actualValue = ((List) result.get(Constants.DATA_LIST)).toArray(new ProjectWorkerGroup[0]); @@ -126,20 +190,8 @@ public class ProjectWorkerGroupRelationServiceTest { return Lists.newArrayList("default"); } - private User getGeneralUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; - } - - private User getAdminUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.ADMIN_USER); - loginUser.setUserName("userName"); - loginUser.setId(1); - return loginUser; + private List getDiffWorkerGroups() { + return Lists.newArrayList("default", "new"); } private Project getProject() { @@ -158,4 +210,20 @@ public class ProjectWorkerGroupRelationServiceTest { projectWorkerGroup.setWorkerGroup("default"); return projectWorkerGroup; } + + private ProjectWorkerGroup getDiffProjectWorkerGroup() { + ProjectWorkerGroup projectWorkerGroup = new ProjectWorkerGroup(); + projectWorkerGroup.setId(2); + projectWorkerGroup.setProjectCode(projectCode); + projectWorkerGroup.setWorkerGroup("new"); + return projectWorkerGroup; + } + + private TaskDefinition getTaskDefinitionWithDiffWorkerGroup() { + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setProjectCode(projectCode); + taskDefinition.setId(1); + taskDefinition.setWorkerGroup("new"); + return taskDefinition; + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java index e77eab8029..ff8ca6ba2c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java @@ -17,13 +17,19 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_DEFINITION_UPDATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest; import org.apache.dolphinscheduler.api.dto.task.TaskUpdateRequest; @@ -75,13 +81,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class TaskDefinitionServiceImplTest { @InjectMocks @@ -114,9 +124,6 @@ public class TaskDefinitionServiceImplTest { @Mock private ProcessTaskRelationMapper processTaskRelationMapper; - @Mock - private TaskPluginManager taskPluginManager; - @Mock private ProcessTaskRelationService processTaskRelationService; @@ -155,61 +162,73 @@ public class TaskDefinitionServiceImplTest { @Test public void createTaskDefinition() { - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, result)) - .thenReturn(true); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); + Map result = new HashMap<>(); + when(projectService.hasProjectAndWritePerm(user, project, result)) + .thenReturn(true); - String createTaskDefinitionJson = - "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" - + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," - + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" - + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," - + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," - + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," - + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; - Map relation = taskDefinitionService - .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + String createTaskDefinitionJson = + "[{\"name\":\"detail_up\",\"description\":\"\",\"taskType\":\"SHELL\",\"taskParams\":" + + "\"{\\\"resourceList\\\":[],\\\"localParams\\\":[{\\\"prop\\\":\\\"datetime\\\",\\\"direct\\\":\\\"IN\\\"," + + "\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${system.datetime}\\\"}],\\\"rawScript\\\":" + + "\\\"echo ${datetime}\\\",\\\"conditionResult\\\":\\\"{\\\\\\\"successNode\\\\\\\":[\\\\\\\"\\\\\\\"]," + + "\\\\\\\"failedNode\\\\\\\":[\\\\\\\"\\\\\\\"]}\\\",\\\"dependence\\\":{}}\",\"flag\":0,\"taskPriority\":0," + + "\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":0,\"timeoutFlag\":0," + + "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]"; + Map relation = taskDefinitionService + .createTaskDefinition(user, PROJECT_CODE, createTaskDefinitionJson); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + + } } @Test public void updateTaskDefinition() { - String taskDefinitionJson = getTaskDefinitionJson();; + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); - Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + Project project = getProject(); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS, PROJECT_CODE); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); - Mockito.when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); - Mockito.when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); - Mockito.when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) - .thenReturn(getProcessTaskRelationList2()); - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); - result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - // failure - Mockito.when(processTaskRelationMapper - .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson)); - Assertions.assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), - ((ServiceException) exception).getCode()); + when(processService.isTaskOnline(TASK_CODE)).thenReturn(Boolean.FALSE); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(new TaskDefinition()); + when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1); + when(processTaskRelationLogDao.insert(Mockito.any(ProcessTaskRelationLog.class))).thenReturn(1); + when(processDefinitionMapper.queryByCode(2L)).thenReturn(new ProcessDefinition()); + when(processDefinitionMapper.updateById(Mockito.any(ProcessDefinition.class))).thenReturn(1); + when(processDefinitionLogMapper.insert(Mockito.any(ProcessDefinitionLog.class))).thenReturn(1); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryProcessTaskRelationByTaskCodeAndTaskVersion(TASK_CODE, 0)) + .thenReturn(getProcessTaskRelationList2()); + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(1); + result = taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, taskDefinitionJson); + assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + // failure + when(processTaskRelationMapper + .updateProcessTaskRelationTaskVersion(Mockito.any(ProcessTaskRelation.class))).thenReturn(2); + exception = Assertions.assertThrows(ServiceException.class, + () -> taskDefinitionService.updateTaskDefinition(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson)); + assertEquals(Status.PROCESS_TASK_RELATION_BATCH_UPDATE_ERROR.getCode(), + ((ServiceException) exception).getCode()); + } } @@ -217,72 +236,72 @@ public class TaskDefinitionServiceImplTest { public void queryTaskDefinitionByName() { String taskName = "task"; Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(result); - Mockito.when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) + when(taskDefinitionMapper.queryByName(project.getCode(), PROCESS_DEFINITION_CODE, taskName)) .thenReturn(new TaskDefinition()); Map relation = taskDefinitionService .queryTaskDefinitionByName(user, PROJECT_CODE, PROCESS_DEFINITION_CODE, taskName); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } @Test public void deleteTaskDefinitionByCode() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); // error task definition not find exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error delete single task definition object - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); - Mockito.when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(0); + when(projectService.hasProjectAndWritePerm(user, project, new HashMap<>())).thenReturn(true); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); - Assertions.assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), + assertEquals(Status.DELETE_TASK_DEFINE_BY_CODE_MSG_ERROR.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_DELETE); - Mockito.when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); - Mockito.when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); + when(processTaskRelationMapper.queryDownstreamByTaskCode(TASK_CODE)).thenReturn(new ArrayList<>()); + when(taskDefinitionMapper.deleteByCode(TASK_CODE)).thenReturn(1); Assertions.assertDoesNotThrow(() -> taskDefinitionService.deleteTaskDefinitionByCode(user, TASK_CODE)); } @Test public void switchVersion() { Project project = getProject(); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, PROJECT_CODE); - Mockito.when( + when( projectService.checkProjectAndAuth(user, project, PROJECT_CODE, WORKFLOW_SWITCH_TO_THIS_VERSION)) - .thenReturn(result); + .thenReturn(result); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, VERSION)) .thenReturn(new TaskDefinitionLog()); TaskDefinition taskDefinition = new TaskDefinition(); taskDefinition.setProjectCode(PROJECT_CODE); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)) + when(taskDefinitionMapper.queryByCode(TASK_CODE)) .thenReturn(taskDefinition); - Mockito.when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); + when(taskDefinitionMapper.updateById(new TaskDefinitionLog())).thenReturn(1); Map relation = taskDefinitionService .switchVersion(user, PROJECT_CODE, TASK_CODE, VERSION); - Assertions.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, relation.get(Constants.STATUS)); } private void putMsg(Map result, Status status, Object... statusParams) { @@ -331,7 +350,7 @@ public class TaskDefinitionServiceImplTest { @Test public void genTaskCodeList() { Map genTaskCodeList = taskDefinitionService.genTaskCodeList(10); - Assertions.assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, genTaskCodeList.get(Constants.STATUS)); } @Test @@ -348,31 +367,31 @@ public class TaskDefinitionServiceImplTest { taskMainInfo.setUpstreamTaskName("4"); taskMainInfoIPage.setRecords(Collections.singletonList(taskMainInfo)); taskMainInfoIPage.setTotal(10L); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, TASK_DEFINITION)) .thenReturn(checkResult); - Mockito.when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), + when(taskDefinitionMapper.queryDefineListPaging(Mockito.any(Page.class), Mockito.anyLong(), Mockito.isNull(), Mockito.anyString(), Mockito.isNull())) - .thenReturn(taskMainInfoIPage); - Mockito.when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) + .thenReturn(taskMainInfoIPage); + when(taskDefinitionMapper.queryDefineListByCodeList(PROJECT_CODE, Collections.singletonList(3L))) .thenReturn(Collections.singletonList(taskMainInfo)); Result result = taskDefinitionService.queryTaskDefinitionListPaging(user, PROJECT_CODE, null, null, null, pageNo, pageSize); - Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test public void testReleaseTaskDefinition() { - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); Project project = getProject(); // check task dose not exist Map result = new HashMap<>(); putMsg(result, Status.TASK_DEFINE_NOT_EXIST, TASK_CODE); - Mockito.when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); + when(projectService.checkProjectAndAuth(user, project, PROJECT_CODE, null)).thenReturn(result); Map map = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); + assertEquals(Status.TASK_DEFINE_NOT_EXIST, map.get(Constants.STATUS)); // process definition offline putMsg(result, Status.SUCCESS); @@ -384,23 +403,23 @@ public class TaskDefinitionServiceImplTest { "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}}"; taskDefinition.setTaskParams(params); taskDefinition.setTaskType("SHELL"); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(taskDefinition); - Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(TASK_CODE, taskDefinition.getVersion())) .thenReturn(taskDefinitionLog); Map offlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.OFFLINE); - Assertions.assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, offlineTaskResult.get(Constants.STATUS)); // process definition online, resource exist Map onlineTaskResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.ONLINE); - Assertions.assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); + assertEquals(Status.SUCCESS, onlineTaskResult.get(Constants.STATUS)); // release error code Map failResult = taskDefinitionService.releaseTaskDefinition(user, PROJECT_CODE, TASK_CODE, ReleaseState.getEnum(2)); - Assertions.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); + assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); } @Test @@ -410,133 +429,127 @@ public class TaskDefinitionServiceImplTest { taskCreateRequest.setWorkflowCode(PROCESS_DEFINITION_CODE); // error process definition not find - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROCESS_DEFINE_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition taskCreateRequest.setTaskParams(TASK_PARAMETER); - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_CREATE); - exception = Assertions.assertThrows(ServiceException.class, + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); - // error create task definition object - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); - // error sync to task definition log - Mockito.when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error create task definition object + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Mockito.when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), - isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); - Assertions.assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + // error sync to task definition log + when(taskDefinitionMapper.insert(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + when(processDefinitionService.updateSingleProcessDefinition(isA(User.class), isA(Long.class), + isA(WorkflowUpdateRequest.class))).thenReturn(getProcessDefinition()); + assertDoesNotThrow(() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); + } } @Test public void testUpdateTaskDefinitionV2() { TaskUpdateRequest taskUpdateRequest = new TaskUpdateRequest(); + TaskDefinition taskDefinition = getTaskDefinition(); + Project project = getProject(); // error task definition not exists - exception = Assertions.assertThrows(ServiceException.class, + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_EXISTS, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_EXISTS.getCode(), ((ServiceException) exception).getCode()); // error project not find - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(isA(Long.class))).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) - .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(projectMapper.queryByCode(isA(Long.class))).thenReturn(project); + doThrow(new ServiceException(Status.PROJECT_NOT_EXIST)).when(projectService) + .checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); + assertThrowsServiceException(Status.PROJECT_NOT_EXIST, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROJECT_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), - TASK_DEFINITION_UPDATE); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, TASK_DEFINITION_UPDATE); - // error task definition already online - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID.getCode(), - ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(false); + assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + } - // error task definition nothing update - Mockito.when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.TASK_DEFINITION_NOT_CHANGE.getCode(), ((ServiceException) exception).getCode()); + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + // error task definition nothing update + when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); + assertThrowsServiceException(Status.TASK_DEFINITION_NOT_CHANGE, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition version invalid - taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.DATA_IS_NOT_VALID.getCode(), ((ServiceException) exception).getCode()); + // error task definition version invalid + taskUpdateRequest.setTaskPriority(String.valueOf(Priority.HIGH)); + assertThrowsServiceException(Status.DATA_IS_NOT_VALID, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition update effect number - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.UPDATE_TASK_DEFINITION_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition update effect number + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(VERSION); + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(0); + assertThrowsServiceException(Status.UPDATE_TASK_DEFINITION_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // error task definition log insert - Mockito.when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); - exception = Assertions.assertThrows(ServiceException.class, - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - Assertions.assertEquals(Status.CREATE_TASK_DEFINITION_LOG_ERROR.getCode(), - ((ServiceException) exception).getCode()); + // error task definition log insert + when(taskDefinitionMapper.updateById(isA(TaskDefinition.class))).thenReturn(1); + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(0); + assertThrowsServiceException(Status.CREATE_TASK_DEFINITION_LOG_ERROR, + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - // success - Mockito.when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); - // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService - Mockito.when( - processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), - isA(Boolean.class), - isA(TaskRelationUpdateUpstreamRequest.class))) - .thenReturn(getProcessTaskRelationList()); - Assertions.assertDoesNotThrow( - () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); + // success + when(taskDefinitionLogMapper.insert(isA(TaskDefinitionLog.class))).thenReturn(1); + // we do not test updateUpstreamTaskDefinition, because it should be tested in processTaskRelationService + when( + processTaskRelationService.updateUpstreamTaskDefinitionWithSyncDag(isA(User.class), isA(Long.class), + isA(Boolean.class), + isA(TaskRelationUpdateUpstreamRequest.class))) + .thenReturn(getProcessTaskRelationList()); + Assertions.assertDoesNotThrow( + () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); - TaskDefinition taskDefinition = - taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); - Assertions.assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + taskDefinition = + taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest); + assertEquals(getTaskDefinition().getVersion() + 1, taskDefinition.getVersion()); + } } @Test @@ -549,28 +562,28 @@ public class TaskDefinitionServiceImplTest { ArrayList taskDefinitionLogs = new ArrayList<>(); taskDefinitionLogs.add(taskDefinitionLog); Integer version = 1; - Mockito.when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); + when(processDefinitionMapper.queryByCode(isA(long.class))).thenReturn(processDefinition); // saveProcessDefine - Mockito.when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); - Mockito.when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); - Mockito.when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefineLogMapper.queryMaxVersionForDefinition(isA(long.class))).thenReturn(version); + when(processDefineLogMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); + when(processDefinitionMapper.insert(isA(ProcessDefinitionLog.class))).thenReturn(1); int insertVersion = processServiceImpl.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE); - Mockito.when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) + when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.TRUE)) .thenReturn(insertVersion); - Assertions.assertEquals(insertVersion, version + 1); + assertEquals(insertVersion, version + 1); // saveTaskRelation List processTaskRelationLogList = getProcessTaskRelationLogList(); - Mockito.when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) + when(processTaskRelationMapper.queryByProcessCode(eq(processDefinition.getCode()))) .thenReturn(processTaskRelationList); - Mockito.when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); - Mockito.when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationMapper.batchInsert(isA(List.class))).thenReturn(1); + when(processTaskRelationLogMapper.batchInsert(isA(List.class))).thenReturn(1); int insertResult = processServiceImpl.saveTaskRelation(loginUser, processDefinition.getProjectCode(), processDefinition.getCode(), insertVersion, processTaskRelationLogList, taskDefinitionLogs, Boolean.TRUE); - Assertions.assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); + assertEquals(Constants.EXIT_CODE_SUCCESS, insertResult); Assertions.assertDoesNotThrow( () -> taskDefinitionService.updateDag(loginUser, processDefinition.getCode(), processTaskRelationList, taskDefinitionLogs)); @@ -581,55 +594,60 @@ public class TaskDefinitionServiceImplTest { // error task definition not exists exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); + assertEquals(Status.TASK_DEFINE_NOT_EXIST.getCode(), ((ServiceException) exception).getCode()); // error task definition not exists - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(getTaskDefinition()); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) .checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); exception = Assertions.assertThrows(ServiceException.class, () -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); - Assertions.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), + assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM.getCode(), ((ServiceException) exception).getCode()); // success - Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION); Assertions.assertDoesNotThrow(() -> taskDefinitionService.getTaskDefinition(user, TASK_CODE)); } @Test public void testUpdateTaskWithUpstream() { + try ( + MockedStatic taskPluginManagerMockedStatic = + Mockito.mockStatic(TaskPluginManager.class)) { + taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + .thenReturn(true); + String taskDefinitionJson = getTaskDefinitionJson(); + TaskDefinition taskDefinition = getTaskDefinition(); + taskDefinition.setFlag(Flag.NO); + TaskDefinition taskDefinitionSecond = getTaskDefinition(); + taskDefinitionSecond.setCode(5); - String taskDefinitionJson = getTaskDefinitionJson(); - TaskDefinition taskDefinition = getTaskDefinition(); - taskDefinition.setFlag(Flag.NO); - TaskDefinition taskDefinitionSecond = getTaskDefinition(); - taskDefinitionSecond.setCode(5); + user.setUserType(UserType.ADMIN_USER); + when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); + when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); + when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); + when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); + when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); + when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); - user.setUserType(UserType.ADMIN_USER); - Mockito.when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(getProject()); - Mockito.when(projectService.hasProjectAndWritePerm(user, getProject(), new HashMap<>())).thenReturn(true); - Mockito.when(taskDefinitionMapper.queryByCode(TASK_CODE)).thenReturn(taskDefinition); - Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true); - Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(TASK_CODE)).thenReturn(1); - Mockito.when(taskDefinitionMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(taskDefinitionLogMapper.insert(Mockito.any())).thenReturn(1); + when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) + .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - Mockito.when(taskDefinitionMapper.queryByCodeList(Mockito.anySet())) - .thenReturn(Arrays.asList(taskDefinition, taskDefinitionSecond)); - - Mockito.when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) - .thenReturn(getProcessTaskRelationListV2()); - Mockito.when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)).thenReturn(getProcessDefinition()); - Mockito.when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); - Mockito.when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); - Mockito.when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); - // success - Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, - taskDefinitionJson, UPSTREAM_CODE); - Assertions.assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); - user.setUserType(UserType.GENERAL_USER); + when(processTaskRelationMapper.queryUpstreamByCode(PROJECT_CODE, TASK_CODE)) + .thenReturn(getProcessTaskRelationListV2()); + when(processDefinitionMapper.queryByCode(PROCESS_DEFINITION_CODE)) + .thenReturn(getProcessDefinition()); + when(processTaskRelationMapper.batchInsert(Mockito.anyList())).thenReturn(1); + when(processTaskRelationMapper.updateById(Mockito.any())).thenReturn(1); + when(processTaskRelationLogDao.batchInsert(Mockito.anyList())).thenReturn(2); + // success + Map successMap = taskDefinitionService.updateTaskWithUpstream(user, PROJECT_CODE, TASK_CODE, + taskDefinitionJson, UPSTREAM_CODE); + assertEquals(Status.SUCCESS, successMap.get(Constants.STATUS)); + user.setUserType(UserType.GENERAL_USER); + } } private String getTaskDefinitionJson() { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java index aca0d80a6f..dd7acb16d5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServiceException; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.FORCED_SUCCESS; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_INSTANCE; import static org.mockito.ArgumentMatchers.any; @@ -25,7 +26,6 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.taskInstance.TaskInstanceRemoveCacheResponse; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; @@ -35,15 +35,16 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -65,7 +66,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.springframework.boot.test.context.SpringBootTest; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -74,7 +74,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -@SpringBootTest(classes = ApiApplicationServer.class) public class TaskInstanceServiceTest { @InjectMocks @@ -100,6 +99,8 @@ public class TaskInstanceServiceTest { @Mock TaskInstanceDao taskInstanceDao; + @Mock + ProcessInstanceDao workflowInstanceDao; @Test public void queryTaskListPaging() { @@ -324,6 +325,7 @@ public class TaskInstanceServiceTest { private TaskInstance getTaskInstance() { TaskInstance taskInstance = new TaskInstance(); taskInstance.setId(1); + taskInstance.setProjectCode(1L); taskInstance.setName("test_task_instance"); taskInstance.setStartTime(new Date()); taskInstance.setEndTime(new Date()); @@ -343,64 +345,69 @@ public class TaskInstanceServiceTest { } @Test - public void testForceTaskSuccess() { + public void testForceTaskSuccess_withNoPermission() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doThrow(new ServiceException(Status.USER_NO_OPERATION_PROJECT_PERM)).when(projectService) + .checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + assertThrowsServiceException(Status.USER_NO_OPERATION_PROJECT_PERM, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withTaskInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.empty()); + assertThrowsServiceException(Status.TASK_INSTANCE_NOT_FOUND, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFound() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())).thenReturn(Optional.empty()); + + assertThrowsServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); + } + + @Test + public void testForceTaskSuccess_withWorkflowInstanceNotFinished() { User user = getAdminUser(); long projectCode = 1L; - Project project = getProject(projectCode); - int taskId = 1; TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, projectCode, FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); - Map mockSuccess = new HashMap<>(5); - putMsg(mockSuccess, Status.SUCCESS); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); + assertThrowsServiceException( + "The workflow instance is not finished: " + processInstance.getState() + + " cannot force start task instance", + () -> taskInstanceService.forceTaskSuccess(user, projectCode, task.getId())); + } - // user auth failed - Map mockFailure = new HashMap<>(5); - putMsg(mockFailure, Status.USER_NO_OPERATION_PROJECT_PERM, user.getUserName(), projectCode); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockFailure); - Result authFailRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertNotSame(Status.SUCCESS.getCode(), authFailRes.getCode()); - - // test task not found - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(mockSuccess); - when(taskInstanceMapper.selectById(Mockito.anyInt())).thenReturn(null); - TaskDefinition taskDefinition = new TaskDefinition(); - taskDefinition.setProjectCode(projectCode); - when(taskDefinitionMapper.queryByCode(task.getTaskCode())).thenReturn(taskDefinition); - Result taskNotFoundRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), taskNotFoundRes.getCode().intValue()); - - // test task instance state error - task.setState(TaskExecutionStatus.SUCCESS); - when(taskInstanceMapper.selectById(1)).thenReturn(task); - Map result = new HashMap<>(); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result taskStateErrorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.TASK_INSTANCE_STATE_OPERATION_ERROR.getCode(), - taskStateErrorRes.getCode().intValue()); - - // test error - task.setState(TaskExecutionStatus.FAILURE); - when(taskInstanceMapper.updateById(task)).thenReturn(0); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result errorRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.FORCE_TASK_SUCCESS_ERROR.getCode(), errorRes.getCode().intValue()); - - // test success - task.setState(TaskExecutionStatus.FAILURE); - task.setEndTime(null); - when(taskInstanceMapper.updateById(task)).thenReturn(1); - putMsg(result, Status.SUCCESS, projectCode); - when(projectMapper.queryByCode(projectCode)).thenReturn(project); - when(projectService.checkProjectAndAuth(user, project, projectCode, FORCED_SUCCESS)).thenReturn(result); - Result successRes = taskInstanceService.forceTaskSuccess(user, projectCode, taskId); - Assertions.assertEquals(Status.SUCCESS.getCode(), successRes.getCode().intValue()); - Assertions.assertNotNull(task.getEndTime()); + @Test + public void testForceTaskSuccess_withTaskInstanceNotFinished() { + User user = getAdminUser(); + TaskInstance task = getTaskInstance(); + ProcessInstance processInstance = getProcessInstance(); + processInstance.setState(WorkflowExecutionStatus.FAILURE); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, task.getProjectCode(), FORCED_SUCCESS); + when(taskInstanceDao.queryOptionalById(task.getId())).thenReturn(Optional.of(task)); + when(workflowInstanceDao.queryOptionalById(task.getProcessInstanceId())) + .thenReturn(Optional.of(processInstance)); + assertThrowsServiceException( + Status.TASK_INSTANCE_STATE_OPERATION_ERROR, + () -> taskInstanceService.forceTaskSuccess(user, task.getProjectCode(), task.getId())); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java index 14e1ed956b..b8161af13d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowsServi import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_CREATE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_UPDATE; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -29,9 +30,11 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.ServiceTestUtil; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.User; @@ -42,6 +45,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -53,6 +57,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -121,9 +126,23 @@ public class EnvironmentServiceTest { when(environmentMapper.insert(any(Environment.class))).thenReturn(1); when(relationMapper.insert(any(EnvironmentWorkerGroupRelation.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); assertDoesNotThrow( () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + when(environmentMapper.insert(any(Environment.class))).thenReturn(-1); + assertThrowsServiceException(Status.CREATE_ENVIRONMENT_ERROR, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + + try (MockedStatic ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { + when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class); + + assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, + () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); + } } @Test @@ -156,25 +175,54 @@ public class EnvironmentServiceTest { assertThrowsServiceException(Status.ENVIRONMENT_NAME_EXISTS, () -> environmentService .updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), getDesc(), workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(-1); + assertThrowsServiceException(Status.UPDATE_ENVIRONMENT_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + workerGroups)); + when(environmentMapper.update(any(Environment.class), any(Wrapper.class))).thenReturn(1); + + assertThrowsServiceException(Status.DESCRIPTION_TOO_LONG_ERROR, + () -> environmentService.updateEnvironmentByCode(adminUser, 2L, environmentName, getConfig(), + ServiceTestUtil.randomStringWithLengthN(512), workerGroups)); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", workerGroups)); + + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); + + when(relationMapper.queryByEnvironmentCode(any())) + .thenReturn(Collections.singletonList(getEnvironmentWorkerGroup())); + assertDoesNotThrow(() -> environmentService.updateEnvironmentByCode(adminUser, 1L, "testName", "test", "test", + "")); } @Test public void testQueryAllEnvironmentList() { + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, + 1, environmentServiceLogger)).thenReturn(Collections.emptySet()); + Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + assertEquals(0, ((List) result.get(Constants.DATA_LIST)).size()); + Set ids = new HashSet<>(); ids.add(1); when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, 1, environmentServiceLogger)).thenReturn(ids); when(environmentMapper.selectBatchIds(ids)).thenReturn(Lists.newArrayList(getEnvironment())); - Map result = environmentService.queryAllEnvironmentList(getAdminUser()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List list = (List) (result.get(Constants.DATA_LIST)); Assertions.assertEquals(1, list.size()); + + when(environmentMapper.selectBatchIds(ids)).thenReturn(Collections.emptyList()); + result = environmentService.queryAllEnvironmentList(getAdminUser()); + Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + list = (List) (result.get(Constants.DATA_LIST)); + Assertions.assertEquals(0, list.size()); } @Test @@ -186,9 +234,27 @@ public class EnvironmentServiceTest { .thenReturn(page); Result result = environmentService.queryEnvironmentListPaging(getAdminUser(), 1, 10, environmentName); - logger.info(result.toString()); PageInfo pageInfo = (PageInfo) result.getData(); Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); + + assertDoesNotThrow( + () -> environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName)); + + when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition( + AuthorizationType.ENVIRONMENT, + 1, + environmentServiceLogger)).thenReturn(Collections.singleton(10)); + when(environmentMapper.queryEnvironmentListPagingByIds(any(Page.class), any(List.class), any(String.class))) + .thenReturn(page); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(1, ((PageInfo) result.getData()).getTotalList().size()); + + page.setRecords(Collections.emptyList()); + page.setTotal(0); + result = environmentService.queryEnvironmentListPaging(getGeneralUser(), 1, 10, environmentName); + assertEquals(0, result.getCode()); + assertEquals(0, ((PageInfo) result.getData()).getTotalList().size()); } @Test @@ -239,6 +305,10 @@ public class EnvironmentServiceTest { result = environmentService.deleteEnvironmentByCode(loginUser, 1L); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + when(environmentMapper.deleteByCode(1L)).thenReturn(-1); + result = environmentService.deleteEnvironmentByCode(loginUser, 1L); + Assertions.assertEquals(Status.DELETE_ENVIRONMENT_ERROR, result.get(Constants.STATUS)); } @Test @@ -251,6 +321,9 @@ public class EnvironmentServiceTest { result = environmentService.verifyEnvironment(environmentName); logger.info(result.toString()); Assertions.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + + when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(null); + assertDoesNotThrow(() -> environmentService.verifyEnvironment(environmentName)); } private Environment getEnvironment() { @@ -264,6 +337,13 @@ public class EnvironmentServiceTest { return environment; } + private EnvironmentWorkerGroupRelation getEnvironmentWorkerGroup() { + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(1L); + relation.setWorkerGroup("new_worker_group"); + return relation; + } + /** * create an environment description */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index bca8a69a16..da5ea88c83 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -92,4 +92,24 @@ public class CheckUtilsTest { Assertions.assertTrue(CheckUtils.checkPhone("17362537263")); } + /** + * check file path + */ + @Test + public void testCheckFilePath() { + // true + Assertions.assertTrue(CheckUtils.checkFilePath("/")); + Assertions.assertTrue(CheckUtils.checkFilePath("xx/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/xx")); + Assertions.assertTrue(CheckUtils.checkFilePath("14567134578654")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/")); + Assertions.assertTrue(CheckUtils.checkFilePath("/admin/root/1531531..13513/153135..")); + // false + Assertions.assertFalse(CheckUtils.checkFilePath(null)); + Assertions.assertFalse(CheckUtils.checkFilePath("file://xxx/ss")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss;/dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss && /dasd/123")); + Assertions.assertFalse(CheckUtils.checkFilePath("/xxx/ss || /dasd/123")); + } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java new file mode 100644 index 0000000000..f33571d309 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ServiceTestUtil.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.nio.charset.StandardCharsets; +import java.util.Random; + +public class ServiceTestUtil { + + public static String randomStringWithLengthN(int n) { + byte[] bitArray = new byte[n]; + new Random().nextBytes(bitArray); + return new String(bitArray, StandardCharsets.UTF_8); + } + + private static User getUser(Integer userId, String userName, UserType userType) { + User user = new User(); + user.setUserType(userType); + user.setId(userId); + user.setUserName(userName); + return user; + } + + public static User getAdminUser() { + return getUser(1, "admin", UserType.ADMIN_USER); + } + public static User getGeneralUser() { + return getUser(10, "user", UserType.GENERAL_USER); + } +} diff --git a/dolphinscheduler-api/src/test/resources/logback-spring.xml b/dolphinscheduler-api/src/test/resources/logback-spring.xml new file mode 100644 index 0000000000..9159c3b02d --- /dev/null +++ b/dolphinscheduler-api/src/test/resources/logback-spring.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-api.log + + ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + 50GB + true + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriber.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java similarity index 74% rename from dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriber.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java index 07202237b2..5411e4cbc1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditSubscriber.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/CommonConfiguration.java @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.api.audit; +package org.apache.dolphinscheduler.common; -public interface AuditSubscriber { +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; - /** - * process the audit message - * - * @param message - */ - void execute(AuditMessage message); +@Configuration +@ComponentScan("org.apache.dolphinscheduler.common") +public class CommonConfiguration { } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java index 742e745fe4..620f74ef95 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java @@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j; * This class will get the property by the priority of the following: env > jvm > properties. */ @Slf4j -public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate { +public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate { private static final Map>> configValueMap = new ConcurrentHashMap<>(); - public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) { - super(propertyAbsolutePath); + private ImmutablePropertyDelegate immutablePropertyDelegate; + + private ImmutableYamlDelegate immutableYamlDelegate; + + public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate, + ImmutableYamlDelegate immutableYamlDelegate) { + this.immutablePropertyDelegate = immutablePropertyDelegate; + this.immutableYamlDelegate = immutableYamlDelegate; } @Override @@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate return value; } value = getConfigValueFromProperties(key); + if (value.isPresent()) { + log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + k, value.get().getActualKey(), value.get().getValue()); + return value; + } + value = getConfigValueFromYaml(key); value.ifPresent( - stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}", k, stringConfigValue.getActualKey(), stringConfigValue.getValue())); return value; }); @@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate @Override public Set getPropertyKeys() { Set propertyKeys = new HashSet<>(); - propertyKeys.addAll(super.getPropertyKeys()); + propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys()); + propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys()); propertyKeys.addAll(System.getProperties().stringPropertyNames()); propertyKeys.addAll(System.getenv().keySet()); return propertyKeys; @@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate } private Optional> getConfigValueFromProperties(String key) { - String value = super.get(key); + String value = this.immutablePropertyDelegate.get(key); + if (value != null) { + return Optional.of(ConfigValue.fromProperties(key, value)); + } + return Optional.empty(); + } + + private Optional> getConfigValueFromYaml(String key) { + String value = this.immutableYamlDelegate.get(key); if (value != null) { return Optional.of(ConfigValue.fromProperties(key, value)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java index b58735afb0..4a0c192210 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java @@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate { } catch (IOException e) { log.error("Load property: {} error, please check if the file exist under classpath", propertyAbsolutePath, e); - System.exit(1); + throw new RuntimeException(e); } } printProperties(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java new file mode 100644 index 0000000000..5806a20fd7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.config; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.InputStreamResource; + +@Slf4j +public class ImmutableYamlDelegate implements IPropertyDelegate { + + private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml"; + + private final Properties properties; + + public ImmutableYamlDelegate() { + this(REMOTE_LOGGING_YAML_NAME); + } + + public ImmutableYamlDelegate(String... yamlAbsolutePath) { + properties = new Properties(); + // read from classpath + for (String fileName : yamlAbsolutePath) { + try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { + YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); + factory.setResources(new InputStreamResource(fis)); + factory.afterPropertiesSet(); + Properties subProperties = factory.getObject(); + properties.putAll(subProperties); + } catch (IOException e) { + log.error("Load property: {} error, please check if the file exist under classpath", + yamlAbsolutePath, e); + throw new RuntimeException(e); + } + } + printProperties(); + } + + public ImmutableYamlDelegate(Properties properties) { + this.properties = properties; + } + + @Override + public String get(String key) { + return properties.getProperty(key); + } + + @Override + public String get(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } + + @Override + public Set getPropertyKeys() { + return properties.stringPropertyNames(); + } + + private void printProperties() { + properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v)); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 054a9410d5..cc07accc9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -35,6 +35,8 @@ public final class Constants { */ public static final String COMMON_PROPERTIES_PATH = "/common.properties"; + public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; + public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; public static final String FORMAT_S_S_COLON = "%s:%s"; @@ -242,16 +244,16 @@ public final class Constants { */ public static final String HTTP_X_REAL_IP = "X-Real-IP"; - /** - * UTF-8 - */ - public static final String UTF_8 = "UTF-8"; - /** * user name regex */ public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,39}$"); + /** + * file path regex + */ + public static final Pattern REGEX_FILE_PATH = Pattern.compile("^[a-zA-Z0-9_./-]+$"); + /** * read permission */ @@ -683,9 +685,6 @@ public final class Constants { public static final Integer QUERY_ALL_ON_WORKFLOW = 2; public static final Integer QUERY_ALL_ON_TASK = 3; - /** - * remote logging - */ public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable"; public static final String REMOTE_LOGGING_TARGET = "remote.logging.target"; @@ -742,4 +741,8 @@ public final class Constants { * K8S sensitive param */ public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : \"))).*?(?=(\",\\n))"; + + public static final String RELEASE_STATE = "releaseState"; + public static final String EXECUTE_TYPE = "executeType"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java new file mode 100644 index 0000000000..5f046882c1 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditModelType.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import lombok.Getter; + +/** + * Audit Model type + */ +@Getter +public enum AuditModelType { + + PROJECT("Project", null), + PROCESS("Process", PROJECT), + PROCESS_INSTANCE("ProcessInstance", PROCESS), + TASK("Task", PROCESS), + TASK_INSTANCE("TaskInstance", TASK), + SCHEDULE("Schedule", PROCESS), + + RESOURCE("Resource", null), + FOLDER("Folder", RESOURCE), + FILE("File", FOLDER), + UDF_FOLDER("UDFFolder", RESOURCE), + UDF_FILE("UDFFile", UDF_FOLDER), + UDF_FUNCTION("UDFFunction", RESOURCE), + TASK_GROUP("TaskGroup", RESOURCE), + + DATASOURCE("Datasource", null), + + SECURITY("Security", null), + TENANT("Tenant", SECURITY), + USER("User", SECURITY), + ALARM_GROUP("AlarmGroup", SECURITY), + ALARM_INSTANCE("AlarmInstance", SECURITY), + WORKER_GROUP("WorkerGroup", SECURITY), + YARN_QUEUE("YarnQueue", SECURITY), + ENVIRONMENT("Environment", SECURITY), + CLUSTER("Cluster", SECURITY), + K8S_NAMESPACE("K8sNamespace", SECURITY), + TOKEN("Token", SECURITY), + ; + private final String name; + private final AuditModelType parentType; + private final List child = new ArrayList<>(); + + private static final HashMap AUDIT_MODEL_MAP = new HashMap<>(); + private static final List AUDIT_MODEL_TREE_LIST = new ArrayList<>(); + + static { + for (AuditModelType auditModelType : values()) { + AUDIT_MODEL_MAP.put(auditModelType.name, auditModelType); + } + + for (AuditModelType auditModelType : values()) { + if (auditModelType.parentType != null) { + of(auditModelType.parentType.name).child.add(auditModelType); + } else { + AUDIT_MODEL_TREE_LIST.add(auditModelType); + } + } + } + + public static List getAuditModelTreeList() { + return AUDIT_MODEL_TREE_LIST; + } + + public static AuditModelType of(String name) { + if (AUDIT_MODEL_MAP.containsKey(name)) { + return AUDIT_MODEL_MAP.get(name); + } + + throw new IllegalArgumentException("invalid audit operation type name " + name); + } + + AuditModelType(String name, AuditModelType parentType) { + this.name = name; + this.parentType = parentType; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditOperationType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditOperationType.java index bcd78dff56..c4ca1b351e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditOperationType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditOperationType.java @@ -17,46 +17,80 @@ package org.apache.dolphinscheduler.common.enums; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; + +import lombok.Getter; /** * Audit Operation type */ +@Getter public enum AuditOperationType { - CREATE(0, "CREATE"), - READ(1, "READ"), - UPDATE(2, "UPDATE"), - DELETE(3, "DELETE"); + CREATE("Create"), + UPDATE("Update"), + BATCH_DELETE("BatchDelete"), + BATCH_START("BatchStart"), + DELETE("Delete"), + CLOSE("Close"), - private final int code; - private final String enMsg; + RELEASE("Release"), + ONLINE("Online"), + OFFLINE("Offline"), - private static HashMap AUDIT_OPERATION_MAP = new HashMap<>(); + RESUME_PAUSE("ResumePause"), + RESUME_FAILURE("ResumeFailure"), + + IMPORT("Import"), + EXPORT("Export"), + + EXECUTE("Execute"), + START("Start"), + MODIFY("Modify"), + RUN("Run"), + RERUN("Rerun"), + BATCH_RERUN("BatchRerun"), + STOP("Stop"), + KILL("Kill"), + PAUSE("Pause"), + MOVE("Move"), + + SWITCH_STATUS("SwitchStatus"), + SWITCH_VERSION("SwitchVersion"), + DELETE_VERSION("DeleteVersion"), + FORCE_SUCCESS("ForceSuccess"), + RENAME("Rename"), + UPLOAD("Upload"), + AUTHORIZE("Authorize"), + UN_AUTHORIZE("UnAuthorize"), + COPY("Copy"), + ; + + private final String name; + + AuditOperationType(String name) { + this.name = name; + } + + private static final HashMap AUDIT_OPERATION_MAP = new HashMap<>(); static { for (AuditOperationType operationType : AuditOperationType.values()) { - AUDIT_OPERATION_MAP.put(operationType.code, operationType); + AUDIT_OPERATION_MAP.put(operationType.name, operationType); } } - AuditOperationType(int code, String enMsg) { - this.code = code; - this.enMsg = enMsg; + public static List getOperationList() { + return new ArrayList<>(AUDIT_OPERATION_MAP.values()); } - public static AuditOperationType of(int status) { - if (AUDIT_OPERATION_MAP.containsKey(status)) { - return AUDIT_OPERATION_MAP.get(status); + public static AuditOperationType of(String name) { + if (AUDIT_OPERATION_MAP.containsKey(name)) { + return AUDIT_OPERATION_MAP.get(name); } - throw new IllegalArgumentException("invalid audit operation type code " + status); - } - public int getCode() { - return code; - } - - public String getMsg() { - return enMsg; + throw new IllegalArgumentException("invalid audit operation type code " + name); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java deleted file mode 100644 index d422cd7de3..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.enums; - -import java.util.HashMap; - -/** - * Audit Module type - */ -public enum AuditResourceType { - - // TODO: add other audit resource enums - USER_MODULE(0, "USER"), - PROJECT_MODULE(1, "PROJECT"); - - private final int code; - private final String enMsg; - - private static HashMap AUDIT_RESOURCE_MAP = new HashMap<>(); - - static { - for (AuditResourceType auditResourceType : AuditResourceType.values()) { - AUDIT_RESOURCE_MAP.put(auditResourceType.code, auditResourceType); - } - } - - AuditResourceType(int code, String enMsg) { - this.code = code; - this.enMsg = enMsg; - } - - public int getCode() { - return this.code; - } - - public String getMsg() { - return this.enMsg; - } - - public static AuditResourceType of(int status) { - if (AUDIT_RESOURCE_MAP.containsKey(status)) { - return AUDIT_RESOURCE_MAP.get(status); - } - throw new IllegalArgumentException("invalid audit resource type code " + status); - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java index 25d8024474..8aab70f304 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java @@ -40,7 +40,6 @@ public class RemoteLogUtils { @Autowired private RemoteLogService autowiredRemoteLogService; - @PostConstruct private void init() { remoteLogService = autowiredRemoteLogService; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java deleted file mode 100644 index 11786fd5a3..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.process; - -import org.apache.dolphinscheduler.common.enums.HttpParametersType; - -import java.util.Objects; - -public class HttpProperty { - - /** - * key - */ - private String prop; - - /** - * httpParametersType - */ - private HttpParametersType httpParametersType; - - /** - * value - */ - private String value; - - public HttpProperty() { - } - - public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { - this.prop = prop; - this.httpParametersType = httpParametersType; - this.value = value; - } - - /** - * getter method - * - * @return the prop - * @see HttpProperty#prop - */ - public String getProp() { - return prop; - } - - /** - * setter method - * - * @param prop the prop to set - * @see HttpProperty#prop - */ - public void setProp(String prop) { - this.prop = prop; - } - - /** - * getter method - * - * @return the value - * @see HttpProperty#value - */ - public String getValue() { - return value; - } - - /** - * setter method - * - * @param value the value to set - * @see HttpProperty#value - */ - public void setValue(String value) { - this.value = value; - } - - public HttpParametersType getHttpParametersType() { - return httpParametersType; - } - - public void setHttpParametersType(HttpParametersType httpParametersType) { - this.httpParametersType = httpParametersType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - HttpProperty property = (HttpProperty) o; - return Objects.equals(prop, property.prop) - && Objects.equals(value, property.value); - } - - @Override - public int hashCode() { - return Objects.hash(prop, value); - } - - @Override - public String toString() { - return "HttpProperty{" - + "prop='" + prop + '\'' - + ", httpParametersType=" + httpParametersType - + ", value='" + value + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java index f35523b59d..3e75264808 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java @@ -1,4 +1,6 @@ -/** Copyright 2010-2012 Twitter, Inc.*/ +/** + * Copyright 2010-2012 Twitter, Inc. + */ package org.apache.dolphinscheduler.common.utils; @@ -6,66 +8,71 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; + /** - * Rewriting based on Twitter snowflake algorithm + * Rewriting based on Twitter snowflake algorithm */ +@Slf4j public class CodeGenerateUtils { - // start timestamp - private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 - // Each machine generates 32 in the same millisecond - private static final long LOW_DIGIT_BIT = 5L; - private static final long MIDDLE_BIT = 2L; - private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); - // The displacement to the left - private static final long MIDDLE_LEFT = LOW_DIGIT_BIT; - private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT; - private final long machineHash; - private long lowDigit = 0L; - private long recordMillisecond = -1L; + private static final CodeGenerator codeGenerator; - private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); - private static final long SYSTEM_NANOTIME = System.nanoTime(); - - private CodeGenerateUtils() throws CodeGenerateException { + static { try { - this.machineHash = - Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1)); + codeGenerator = new CodeGenerator(InetAddress.getLocalHost().getHostName() + "-" + OSUtils.getProcessID()); } catch (UnknownHostException e) { throw new CodeGenerateException(e.getMessage()); } } - private static CodeGenerateUtils instance = null; - - public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException { - if (instance == null) { - instance = new CodeGenerateUtils(); - } - return instance; + public static long genCode() throws CodeGenerateException { + return codeGenerator.genCode(); } - public synchronized long genCode() throws CodeGenerateException { - long nowtMillisecond = systemMillisecond(); - if (nowtMillisecond < recordMillisecond) { - throw new CodeGenerateException("New code exception because time is set back."); + public static class CodeGenerator { + + // start timestamp + private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00 + // Each machine generates 32 in the same millisecond + private static final long LOW_DIGIT_BIT = 5L; + private static final long MACHINE_BIT = 5L; + private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); + // The displacement to the left + private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MACHINE_BIT; + public final long machineHash; + private long lowDigit = 0L; + private long recordMillisecond = -1L; + + private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); + private static final long SYSTEM_NANOTIME = System.nanoTime(); + + public CodeGenerator(String appName) { + this.machineHash = Math.abs(Objects.hash(appName)) % (1 << MACHINE_BIT); } - if (nowtMillisecond == recordMillisecond) { - lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; - if (lowDigit == 0L) { - while (nowtMillisecond <= recordMillisecond) { - nowtMillisecond = systemMillisecond(); - } + + public synchronized long genCode() throws CodeGenerateException { + long nowtMillisecond = systemMillisecond(); + if (nowtMillisecond < recordMillisecond) { + throw new CodeGenerateException("New code exception because time is set back."); } - } else { - lowDigit = 0L; + if (nowtMillisecond == recordMillisecond) { + lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; + if (lowDigit == 0L) { + while (nowtMillisecond <= recordMillisecond) { + nowtMillisecond = systemMillisecond(); + } + } + } else { + lowDigit = 0L; + } + recordMillisecond = nowtMillisecond; + return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << LOW_DIGIT_BIT | lowDigit; } - recordMillisecond = nowtMillisecond; - return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit; - } - private long systemMillisecond() { - return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + private long systemMillisecond() { + return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + } } public static class CodeGenerateException extends RuntimeException { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index fee1d9a95c..60629576d9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPA import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE; -import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS; import org.apache.commons.io.IOUtils; @@ -207,7 +206,7 @@ public class FileUtils { while ((length = inputStream.read(buffer)) != -1) { output.write(buffer, 0, length); } - return output.toString(UTF_8); + return output.toString(StandardCharsets.UTF_8.name()); } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException(e); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 5c79fff951..e5d2256150 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -36,6 +36,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -143,7 +144,7 @@ public class HttpUtils { } HttpEntity entity = response.getEntity(); - return entity != null ? EntityUtils.toString(entity, Constants.UTF_8) : null; + return entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null; } catch (IOException e) { log.error("Error executing HTTP GET request", e); return null; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 6750b364f9..bfc3af2c58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -196,7 +196,10 @@ public final class JSONUtils { * @return true if valid */ public static boolean checkJsonValid(String json) { + return checkJsonValid(json, true); + } + public static boolean checkJsonValid(String json, Boolean logFlag) { if (Strings.isNullOrEmpty(json)) { return false; } @@ -205,7 +208,8 @@ public final class JSONUtils { objectMapper.readTree(json); return true; } catch (IOException e) { - log.error("check json object valid exception!", e); + if (logFlag) + log.error("check json object valid exception!", e); } return false; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index beca53c3fd..dbfcea2ed8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -183,11 +183,10 @@ public class OSUtils { * * @param userName user name */ - public static void createUserIfAbsent(String userName) { + public static synchronized void createUserIfAbsent(String userName) { // if not exists this user, then create if (!getUserList().contains(userName)) { - boolean isSuccess = createUser(userName); - log.info("create user {} {}", userName, isSuccess ? "success" : "fail"); + createUser(userName); } } @@ -197,13 +196,12 @@ public class OSUtils { * @param userName user name * @return true if creation was successful, otherwise false */ - public static boolean createUser(String userName) { + public static void createUser(String userName) { try { String userGroup = getGroup(); if (StringUtils.isEmpty(userGroup)) { - String errorLog = String.format("%s group does not exist for this operating system.", userGroup); - log.error(errorLog); - return false; + throw new UnsupportedOperationException( + "There is no userGroup exist cannot create tenant, please create userGroupFirst"); } if (SystemUtils.IS_OS_MAC) { createMacUser(userName, userGroup); @@ -212,18 +210,17 @@ public class OSUtils { } else { createLinuxUser(userName, userGroup); } - return true; + log.info("Create tenant {} under userGroup: {} success", userName, userGroup); } catch (Exception e) { - log.error(e.getMessage(), e); + throw new RuntimeException("Create tenant: {} failed", e); } - return false; } /** * create linux user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -237,7 +234,7 @@ public class OSUtils { /** * create mac user (Supports Mac OSX 10.10+) * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -256,7 +253,7 @@ public class OSUtils { /** * create windows user * - * @param userName user name + * @param userName user name * @param userGroup user group * @throws IOException in case of an I/O error */ @@ -304,7 +301,7 @@ public class OSUtils { * get sudo command * * @param tenantCode tenantCode - * @param command command + * @param command command * @return result of sudo execute command */ public static String getSudoCmd(String tenantCode, String command) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8289b14479..82d4de9599 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; -import org.apache.dolphinscheduler.common.config.IPropertyDelegate; import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate; import java.util.HashMap; import java.util.Map; @@ -37,8 +39,10 @@ import com.google.common.base.Strings; public class PropertyUtils { // todo: add another implementation for zookeeper/etcd/consul/xx - private static final IPropertyDelegate propertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + private final ImmutablePriorityPropertyDelegate propertyDelegate = + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml new file mode 100644 index 0000000000..2cb48750a4 --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java index efba923a5a..6333250492 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config; import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test; class ImmutablePriorityPropertyDelegateTest { private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); @Test void getOverrideFromEnv() throws Exception { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index 3871646c95..8cd8ab8e6d 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -17,20 +17,59 @@ package org.apache.dolphinscheduler.common.utils; -import java.util.HashSet; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class CodeGenerateUtilsTest { +class CodeGenerateUtilsTest { @Test - public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { - HashSet existsCode = new HashSet<>(); - for (int i = 0; i < 100; i++) { - Long currentCode = CodeGenerateUtils.getInstance().genCode(); - Assertions.assertFalse(existsCode.contains(currentCode)); + void testNoGenerateDuplicateCode() { + int codeNum = 10000000; + List existsCode = new ArrayList<>(); + for (int i = 0; i < codeNum; i++) { + Long currentCode = CodeGenerateUtils.genCode(); existsCode.add(currentCode); } + Set existsCodeSet = new HashSet<>(existsCode); + // Disallow duplicate code + assertEquals(existsCode.size(), existsCodeSet.size()); + } + + @Test + void testNoGenerateDuplicateCodeWithDifferentAppName() throws UnknownHostException, InterruptedException { + int threadNum = 10; + int codeNum = 1000000; + + final String hostName = InetAddress.getLocalHost().getHostName(); + Map> machineCodes = new ConcurrentHashMap<>(); + CountDownLatch countDownLatch = new CountDownLatch(threadNum); + + for (int i = 0; i < threadNum; i++) { + final int c = i; + new Thread(() -> { + List codes = new ArrayList<>(codeNum); + CodeGenerateUtils.CodeGenerator codeGenerator = new CodeGenerateUtils.CodeGenerator(hostName + "-" + c); + for (int j = 0; j < codeNum; j++) { + codes.add(codeGenerator.genCode()); + } + machineCodes.put(Thread.currentThread().getName(), codes); + countDownLatch.countDown(); + }).start(); + } + countDownLatch.await(); + Set totalCodes = new HashSet<>(); + machineCodes.values().forEach(totalCodes::addAll); + assertEquals(codeNum * threadNum, totalCodes.size()); } } diff --git a/dolphinscheduler-common/src/test/resources/remote-logging.yaml b/dolphinscheduler-common/src/test/resources/remote-logging.yaml new file mode 100644 index 0000000000..cb149a77fe --- /dev/null +++ b/dolphinscheduler-common/src/test/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..32fb807a9b --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DatabaseEnvironmentCondition.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.api; + +import java.util.Arrays; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class DatabaseEnvironmentCondition implements Condition { + + private final String profile; + + public DatabaseEnvironmentCondition(String profile) { + this.profile = profile; + } + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + String[] activeProfiles = context.getEnvironment().getActiveProfiles(); + return Arrays.asList(activeProfiles).contains(profile); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java index 9aea94f77d..f496679ed5 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("h2") -@Configuration -public class H2DaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(H2DatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class H2DaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..894f38bd20 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.h2; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class H2DatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public H2DatabaseEnvironmentCondition() { + super("h2"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..c899dfb43f --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.h2.H2DaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java index 8b37fca67b..5fb3a350ae 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginAutoConfiguration.java @@ -28,14 +28,14 @@ import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("mysql") -@Configuration -public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Configuration(proxyBeanMethods = false) +@Conditional(MysqlDatabaseEnvironmentCondition.class) +public class MysqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2136e1354e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.mysql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class MysqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public MysqlDatabaseEnvironmentCondition() { + super("mysql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..386c80c676 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.mysql.MysqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java similarity index 88% rename from dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java index e57c84fab9..f0467bfd07 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginAutoConfiguration.java @@ -29,14 +29,14 @@ import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonit import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import com.baomidou.mybatisplus.annotation.DbType; -@Profile("postgresql") -@Configuration -public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration { +@Conditional(PostgresqlDatabaseEnvironmentCondition.class) +@Configuration(proxyBeanMethods = false) +public class PostgresqlDaoPluginAutoConfiguration implements DaoPluginConfiguration { @Autowired private DataSource dataSource; diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java new file mode 100644 index 0000000000..2c71ea8442 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDatabaseEnvironmentCondition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.postgresql; + +import org.apache.dolphinscheduler.dao.plugin.api.DatabaseEnvironmentCondition; + +public class PostgresqlDatabaseEnvironmentCondition extends DatabaseEnvironmentCondition { + + public PostgresqlDatabaseEnvironmentCondition() { + super("postgresql"); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..fd6a5f07b9 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.dolphinscheduler.dao.plugin.postgresql.PostgresqlDaoPluginAutoConfiguration diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index 985f5b56b4..e089c8086b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -40,8 +40,8 @@ import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; @Configuration +@ComponentScan("org.apache.dolphinscheduler.dao") @EnableAutoConfiguration -@ComponentScan({"org.apache.dolphinscheduler.dao.plugin"}) @MapperScan(basePackages = "org.apache.dolphinscheduler.dao.mapper", sqlSessionFactoryRef = "sqlSessionFactory") public class DaoConfiguration { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java index 71e3be70c4..24cb022881 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java @@ -29,10 +29,10 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Repository; @Slf4j -@Component +@Repository public class PluginDao { @Autowired diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java index 10023d1547..7c6f76e1da 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AuditLog.java @@ -19,11 +19,16 @@ package org.apache.dolphinscheduler.dao.entity; import java.util.Date; +import lombok.Data; + +import org.springframework.beans.BeanUtils; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +@Data @TableName("t_ds_audit_log") public class AuditLog { @@ -39,19 +44,19 @@ public class AuditLog { private Integer userId; /** - * resource type + * model type */ - private Integer resourceType; + private String modelType; /** * operation type */ - private Integer operation; + private String operationType; /** - * resource id + * model id */ - private Integer resourceId; + private Long modelId; /** * user name @@ -62,53 +67,19 @@ public class AuditLog { /** * operation time */ - private Date time; + private Date createTime; - public Integer getUserId() { - return userId; - } + private String detail; - public void setUserId(Integer userId) { - this.userId = userId; - } + private String description; - public Integer getResourceType() { - return resourceType; - } + private String modelName; - public void setResourceType(Integer resourceType) { - this.resourceType = resourceType; - } + private long latency; - public Integer getOperation() { - return operation; - } - - public void setOperation(Integer operation) { - this.operation = operation; - } - - public Integer getResourceId() { - return resourceId; - } - - public void setResourceId(Integer resourceId) { - this.resourceId = resourceId; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public Date getTime() { - return time; - } - - public void setTime(Date time) { - this.time = time; + public static AuditLog copyNewOne(AuditLog auditLog) { + AuditLog auditLogNew = new AuditLog(); + BeanUtils.copyProperties(auditLog, auditLogNew); + return auditLogNew; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index aa4d3d4782..d52984b61f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -52,6 +52,10 @@ public class ErrorCommand { */ private long processDefinitionCode; + private int processDefinitionVersion; + + private int processInstanceId; + /** * executor id */ @@ -73,7 +77,7 @@ public class ErrorCommand { private FailureStrategy failureStrategy; /** - * warning type + * warning type */ private WarningType warningType; @@ -135,21 +139,26 @@ public class ErrorCommand { public ErrorCommand() { } + public ErrorCommand(Command command, String message) { this.id = command.getId(); this.commandType = command.getCommandType(); this.executorId = command.getExecutorId(); this.processDefinitionCode = command.getProcessDefinitionCode(); + this.processDefinitionVersion = command.getProcessDefinitionVersion(); + this.processInstanceId = command.getProcessInstanceId(); this.commandParam = command.getCommandParam(); + this.taskDependType = command.getTaskDependType(); + this.failureStrategy = command.getFailureStrategy(); this.warningType = command.getWarningType(); this.warningGroupId = command.getWarningGroupId(); this.scheduleTime = command.getScheduleTime(); - this.taskDependType = command.getTaskDependType(); - this.failureStrategy = command.getFailureStrategy(); this.startTime = command.getStartTime(); this.updateTime = command.getUpdateTime(); - this.environmentCode = command.getEnvironmentCode(); this.processInstancePriority = command.getProcessInstancePriority(); + this.workerGroup = command.getWorkerGroup(); + this.tenantCode = command.getTenantCode(); + this.environmentCode = command.getEnvironmentCode(); this.message = message; this.dryRun = command.getDryRun(); this.testFlag = command.getTestFlag(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java index 7d77b3e429..3cc95a76d1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.dao.entity.AuditLog; import org.apache.ibatis.annotations.Param; import java.util.Date; +import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -32,12 +33,11 @@ import com.baomidou.mybatisplus.core.metadata.IPage; public interface AuditLogMapper extends BaseMapper { IPage queryAuditLog(IPage page, - @Param("resourceType") int[] resourceArray, - @Param("operationType") int[] operationType, + @Param("modelTypeList") List modelTypeList, + @Param("operationTypeList") List operationTypeList, @Param("userName") String userName, + @Param("modelName") String modelName, @Param("startDate") Date startDate, @Param("endDate") Date endDate); - String queryResourceNameByType(@Param("resourceType") String resourceType, - @Param("resourceId") Integer resourceId); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index a8490cbef7..8c8314e7cc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -34,8 +34,9 @@ public interface CommandMapper extends BaseMapper { /** * count command state - * @param startTime startTime - * @param endTime endTime + * + * @param startTime startTime + * @param endTime endTime * @param projectCodes projectCodes * @return CommandCount list */ @@ -46,15 +47,15 @@ public interface CommandMapper extends BaseMapper { /** * query command page + * * @return */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); - /** - * query command page by slot - * @return command list - */ - List queryCommandPageBySlot(@Param("limit") int limit, - @Param("masterCount") int masterCount, - @Param("thisMasterSlot") int thisMasterSlot); + List queryCommandByIdSlot(@Param("currentSlotIndex") int currentSlotIndex, + @Param("totalSlot") int totalSlot, + @Param("idStep") int idStep, + @Param("fetchNumber") int fetchNum); + + void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List workflowInstanceIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java index 10a0acf47f..912ef28101 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TriggerRelationMapper.java @@ -36,7 +36,7 @@ public interface TriggerRelationMapper extends BaseMapper { * @param jobId * @return */ - TriggerRelation queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); + List queryByTypeAndJobId(@Param("triggerType") Integer triggerType, @Param("jobId") int jobId); /** * query triggerRelation by code diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 6bc8049c7d..abc12f9aa1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -109,13 +109,6 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId, @Param("resourceIds") int[] resourceIds); - /** - * batch update udf func - * @param udfFuncList udf list - * @return update num - */ - int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); - /** * listAuthorizedUdfByUserId * @param userId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java index 2937957dbd..664b56ee47 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/BaseDao.java @@ -56,6 +56,11 @@ public abstract class BaseDao> return mybatisMapper.selectBatchIds(ids); } + @Override + public List queryAll() { + return mybatisMapper.selectList(null); + } + @Override public List queryByCondition(ENTITY queryCondition) { if (queryCondition == null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java new file mode 100644 index 0000000000..daa52b8318 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/CommandDao.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.dao.entity.Command; + +import java.util.List; + +public interface CommandDao extends IDao { + + /** + * Query command by command id and server slot, return the command which match (commandId / step) %s totalSlot = currentSlotIndex + * + * @param currentSlotIndex current slot index + * @param totalSlot total slot number + * @param idStep id step in db + * @param fetchNum fetch number + * @return command list + */ + List queryCommandByIdSlot(int currentSlotIndex, + int totalSlot, + int idStep, + int fetchNum); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java index c566d9b904..ab77419600 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IDao.java @@ -41,6 +41,11 @@ public interface IDao { */ List queryByIds(Collection ids); + /** + * Query all entities. + */ + List queryAll(); + /** * Query the entity by condition. */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java new file mode 100644 index 0000000000..0b510d15b5 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImpl.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import java.util.List; + +import org.springframework.stereotype.Repository; + +@Repository +public class CommandDaoImpl extends BaseDao implements CommandDao { + + public CommandDaoImpl(CommandMapper commandMapper) { + super(commandMapper); + } + + @Override + public List queryCommandByIdSlot(int currentSlotIndex, int totalSlot, int idStep, int fetchNum) { + return mybatisMapper.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchNum); + } + +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml index 09ba57075f..63adb57c34 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.xml @@ -19,10 +19,10 @@ - id, user_id, resource_type, operation, resource_id, time + id, user_id, model_type, operation_type, model_id, model_name, create_time, detail, description, latency - ${alias}.id, ${alias}.user_id, ${alias}.resource_type, ${alias}.operation, ${alias}.resource_id, ${alias}.time + ${alias}.id, ${alias}.user_id, ${alias}.model_type, ${alias}.operation_type, ${alias}.model_id, ${alias}.model_name, ${alias}.create_time, ${alias}.detail, ${alias}.description, ${alias}.latency - diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index c950f66413..16f7c05f25 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -40,11 +40,18 @@ limit #{limit} offset #{offset} - select * from t_ds_command - where id % #{masterCount} = #{thisMasterSlot} + where (id / #{idStep}) % #{totalSlot} = #{currentSlotIndex} order by process_instance_priority, id asc - limit #{limit} + limit #{fetchNumber} + + delete from t_ds_command + where process_instance_id in + + #{i} + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index 24cb8de956..84f9fd9708 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -171,18 +171,6 @@ - - - update t_ds_udfs - - resource_name=#{udf.resourceName}, - update_time=#{udf.updateTime} - - - id=#{udf.id} - - -